soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

chore(8.0): final pre-RC1 sweep — API consistency, named errors, orphans, zero-cast codebase

Browse source
This commit is contained in:
David Snelling 2026-06-11 14:51:00 -07:00
parent 970e08c466
commit 1f7e365a4e
237 changed files with 1951 additions and 49413 deletions
Download patch file Download diff file Expand all files Collapse all files

20
src/aggregation/AggregationIndex.ts
View file

@ -312,7 +312,7 @@ export class AggregationIndex {
*/
async init(): Promise<void> {
// Load persisted definitions
const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY) as any
const savedDefs = await this.storage.getMetadata(DEFINITIONS_KEY)
if (savedDefs && typeof savedDefs === 'object' && savedDefs.definitions) {
const defs = savedDefs.definitions as Array<AggregateDefinition & { _hash?: string }>
@ -322,7 +322,7 @@ export class AggregationIndex {
const savedHash = def._hash || ''
// Load persisted state
const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) as any
const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`)
if (stateData && stateData.groups && savedHash === currentHash) {
// Definition unchanged — load state
const groupMap = new Map<string, AggregateGroupState>()
@ -349,11 +349,13 @@ export class AggregationIndex {
// Restore native provider state from persistence
if (this.nativeProvider?.restoreState) {
const nativeState = await this.storage.getMetadata('__aggregation_native_state__') as any
const nativeState = await this.storage.getMetadata('__aggregation_native_state__')
if (nativeState && typeof nativeState === 'string') {
this.nativeProvider.restoreState(nativeState)
} else if (nativeState && typeof nativeState === 'object' && nativeState.data) {
this.nativeProvider.restoreState(nativeState.data)
// flush() persists `{ data: serializeState() }`, so `data` is the
// provider's serialized state string.
this.nativeProvider.restoreState(nativeState.data as string)
}
}
}
@ -367,7 +369,7 @@ export class AggregationIndex {
...def,
_hash: this.definitionHashes.get(def.name)
}))
await this.storage.saveMetadata(DEFINITIONS_KEY, { definitions: defsToSave } as any)
await this.storage.saveMetadata(DEFINITIONS_KEY, { definitions: defsToSave })
// Persist dirty states
for (const name of this.dirty) {
@ -376,7 +378,7 @@ export class AggregationIndex {
const groups = Array.from(stateMap.values())
await this.storage.saveMetadata(
`${STATE_KEY_PREFIX}${name}__`,
{ groups } as any
{ groups }
)
}
}
@ -386,7 +388,7 @@ export class AggregationIndex {
const nativeState = this.nativeProvider.serializeState()
await this.storage.saveMetadata(
'__aggregation_native_state__',
{ data: nativeState } as any
{ data: nativeState }
)
}
@ -613,7 +615,7 @@ export class AggregationIndex {
// Apply where filter on group keys
if (params.where && Object.keys(params.where).length > 0) {
if (!matchesMetadataFilter(group.groupKey as any, params.where)) continue
if (!matchesMetadataFilter(group.groupKey, params.where)) continue
}
// Compute result metrics from running state
@ -664,7 +666,7 @@ export class AggregationIndex {
// HAVING: filter groups by computed metric values (post-compute, O(groups), before
// sort/pagination). Reuses the where-operator engine over metrics + `count`.
if (params.having && Object.keys(params.having).length > 0) {
if (!matchesMetadataFilter({ ...metrics, count: totalCount } as any, params.having)) continue
if (!matchesMetadataFilter({ ...metrics, count: totalCount }, params.having)) continue
}
results.push({

262
src/api/ConfigAPI.ts
View file

@ -1,262 +0,0 @@
/**
* Configuration API for Brainy 3.0
* Provides configuration storage with optional encryption
*/
import { SecurityAPI } from './SecurityAPI.js'
import { StorageAdapter } from '../coreTypes.js'
export interface ConfigOptions {
encrypt?: boolean
decrypt?: boolean
}
export interface ConfigEntry {
key: string
value: any
encrypted: boolean
createdAt: number
updatedAt: number
}
export class ConfigAPI {
private security: SecurityAPI
private configCache: Map<string, ConfigEntry> = new Map()
private CONFIG_NOUN_PREFIX = '_config_'
constructor(private storage: StorageAdapter) {
this.security = new SecurityAPI()
}
/**
* Set a configuration value with optional encryption
*/
async set(params: {
key: string
value: any
encrypt?: boolean
}): Promise<void> {
const { key, value, encrypt = false } = params
// Serialize and optionally encrypt the value
let storedValue: any = value
if (typeof value !== 'string') {
storedValue = JSON.stringify(value)
}
if (encrypt) {
storedValue = await this.security.encrypt(storedValue)
}
// Create config entry
const entry: ConfigEntry = {
key,
value: storedValue,
encrypted: encrypt,
createdAt: this.configCache.get(key)?.createdAt || Date.now(),
updatedAt: Date.now()
}
// Store in cache
this.configCache.set(key, entry)
// Persist to storage as NounMetadata
const configId = this.CONFIG_NOUN_PREFIX + key
const nounMetadata = {
noun: 'config' as any,
...entry,
service: 'config',
createdAt: entry.createdAt,
updatedAt: entry.updatedAt
}
await this.storage.saveNounMetadata(configId, nounMetadata)
}
/**
* Get a configuration value with optional decryption
*/
async get(params: {
key: string
decrypt?: boolean
defaultValue?: any
}): Promise<any> {
const { key, decrypt, defaultValue } = params
// Check cache first
let entry = this.configCache.get(key)
// If not in cache, load from storage
if (!entry) {
const configId = this.CONFIG_NOUN_PREFIX + key
const metadata = await this.storage.getNounMetadata(configId)
if (!metadata) {
return defaultValue
}
// Extract ConfigEntry from NounMetadata
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
: ((metadata.createdAt as unknown as number) || Date.now())
const updatedAtVal = typeof metadata.updatedAt === 'object' && metadata.updatedAt !== null && 'seconds' in metadata.updatedAt
? metadata.updatedAt.seconds * 1000 + Math.floor((metadata.updatedAt.nanoseconds || 0) / 1000000)
: ((metadata.updatedAt as unknown as number) || createdAtVal)
entry = {
key: metadata.key as string,
value: metadata.value,
encrypted: metadata.encrypted as boolean,
createdAt: createdAtVal,
updatedAt: updatedAtVal
}
this.configCache.set(key, entry)
}
let value = entry.value
// Decrypt if needed
const shouldDecrypt = decrypt !== undefined ? decrypt : entry.encrypted
if (shouldDecrypt && entry.encrypted && typeof value === 'string') {
value = await this.security.decrypt(value)
}
// Try to parse JSON if it looks like JSON
if (typeof value === 'string' && (value.startsWith('{') || value.startsWith('['))) {
try {
value = JSON.parse(value)
} catch {
// Not JSON, return as string
}
}
return value
}
/**
* Delete a configuration value
*/
async delete(key: string): Promise<void> {
// Remove from cache
this.configCache.delete(key)
// Remove from storage
const configId = this.CONFIG_NOUN_PREFIX + key
await this.storage.deleteNounMetadata(configId)
}
/**
* List all configuration keys
*/
async list(): Promise<string[]> {
// Get all nouns and filter for config entries
const result = await this.storage.getNouns({ pagination: { limit: 10000 } })
const configKeys: string[] = []
for (const noun of result.items) {
if (noun.id.startsWith(this.CONFIG_NOUN_PREFIX)) {
configKeys.push(noun.id.substring(this.CONFIG_NOUN_PREFIX.length))
}
}
return configKeys
}
/**
* Check if a configuration key exists
*/
async has(key: string): Promise<boolean> {
if (this.configCache.has(key)) {
return true
}
const configId = this.CONFIG_NOUN_PREFIX + key
const metadata = await this.storage.getNounMetadata(configId)
return metadata !== null && metadata !== undefined
}
/**
* Clear all configuration
*/
async clear(): Promise<void> {
// Clear cache
this.configCache.clear()
// Clear from storage
const keys = await this.list()
for (const key of keys) {
await this.delete(key)
}
}
/**
* Export all configuration
*/
async export(): Promise<Record<string, ConfigEntry>> {
const keys = await this.list()
const config: Record<string, ConfigEntry> = {}
for (const key of keys) {
const entry = await this.getEntry(key)
if (entry) {
config[key] = entry
}
}
return config
}
/**
* Import configuration
*/
async import(config: Record<string, ConfigEntry>): Promise<void> {
for (const [key, entry] of Object.entries(config)) {
this.configCache.set(key, entry)
const configId = this.CONFIG_NOUN_PREFIX + key
// Convert ConfigEntry to NounMetadata
const nounMetadata = {
noun: 'config' as any,
...entry,
service: 'config',
createdAt: entry.createdAt,
updatedAt: entry.updatedAt
}
await this.storage.saveNounMetadata(configId, nounMetadata)
}
}
/**
* Get raw config entry (without decryption)
*/
private async getEntry(key: string): Promise<ConfigEntry | null> {
if (this.configCache.has(key)) {
return this.configCache.get(key)!
}
const configId = this.CONFIG_NOUN_PREFIX + key
const metadata = await this.storage.getNounMetadata(configId)
if (!metadata) {
return null
}
// Extract ConfigEntry from NounMetadata
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
: ((metadata.createdAt as unknown as number) || Date.now())
const updatedAtVal = typeof metadata.updatedAt === 'object' && metadata.updatedAt !== null && 'seconds' in metadata.updatedAt
? metadata.updatedAt.seconds * 1000 + Math.floor((metadata.updatedAt.nanoseconds || 0) / 1000000)
: ((metadata.updatedAt as unknown as number) || createdAtVal)
const entry: ConfigEntry = {
key: metadata.key as string,
value: metadata.value,
encrypted: metadata.encrypted as boolean,
createdAt: createdAtVal,
updatedAt: updatedAtVal
}
this.configCache.set(key, entry)
return entry
}
}

163
src/api/SecurityAPI.ts
View file

@ -1,163 +0,0 @@
/**
* Security API for Brainy 3.0
* Provides encryption, decryption, hashing, and secure storage
*/
export class SecurityAPI {
private encryptionKey?: Uint8Array
constructor(private config?: { encryptionKey?: string }) {
if (config?.encryptionKey) {
// Use provided key (must be 32 bytes hex string)
this.encryptionKey = this.hexToBytes(config.encryptionKey)
}
}
/**
* Encrypt data using AES-256-CBC
*/
async encrypt(data: string): Promise<string> {
const crypto = await import('../universal/crypto.js')
// Generate or use existing key
const key = this.encryptionKey || crypto.randomBytes(32)
const iv = crypto.randomBytes(16)
const cipher = crypto.createCipheriv('aes-256-cbc', key, iv)
let encrypted = cipher.update(data, 'utf8', 'hex')
encrypted += cipher.final('hex')
// Package encrypted data with metadata
// In production, store keys separately in a key management service
return JSON.stringify({
encrypted,
key: this.bytesToHex(key),
iv: this.bytesToHex(iv),
algorithm: 'aes-256-cbc',
timestamp: Date.now()
})
}
/**
* Decrypt data encrypted with encrypt()
*/
async decrypt(encryptedData: string): Promise<string> {
const crypto = await import('../universal/crypto.js')
try {
const parsed = JSON.parse(encryptedData)
const { encrypted, key: keyHex, iv: ivHex, algorithm } = parsed
if (algorithm && algorithm !== 'aes-256-cbc') {
throw new Error(`Unsupported encryption algorithm: ${algorithm}`)
}
const key = this.hexToBytes(keyHex)
const iv = this.hexToBytes(ivHex)
const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv)
let decrypted = decipher.update(encrypted, 'hex', 'utf8')
decrypted += decipher.final('utf8')
return decrypted
} catch (error) {
throw new Error(`Decryption failed: ${(error as Error).message}`)
}
}
/**
* Create a one-way hash of data (for passwords, etc)
*/
async hash(data: string, algorithm: 'sha256' | 'sha512' = 'sha256'): Promise<string> {
const crypto = await import('../universal/crypto.js')
const hash = crypto.createHash(algorithm)
hash.update(data)
return hash.digest('hex')
}
/**
* Compare data with a hash (for password verification)
*/
async compare(data: string, hash: string, algorithm: 'sha256' | 'sha512' = 'sha256'): Promise<boolean> {
const dataHash = await this.hash(data, algorithm)
return this.constantTimeCompare(dataHash, hash)
}
/**
* Generate a secure random token
*/
async generateToken(bytes: number = 32): Promise<string> {
const crypto = await import('../universal/crypto.js')
const buffer = crypto.randomBytes(bytes)
return this.bytesToHex(buffer)
}
/**
* Derive a key from a password using PBKDF2
* Note: Simplified version using hash instead of PBKDF2 which may not be available
*/
async deriveKey(password: string, salt?: string, iterations: number = 100000): Promise<{
key: string
salt: string
}> {
const crypto = await import('../universal/crypto.js')
const actualSalt = salt || this.bytesToHex(crypto.randomBytes(32))
// Simplified key derivation using repeated hashing
// In production, use a proper PBKDF2 implementation
let derived = password + actualSalt
for (let i = 0; i < Math.min(iterations, 1000); i++) {
const hash = crypto.createHash('sha256')
hash.update(derived)
derived = hash.digest('hex')
}
return {
key: derived,
salt: actualSalt
}
}
/**
* Sign data with HMAC
*/
async sign(data: string, secret?: string): Promise<string> {
const crypto = await import('../universal/crypto.js')
const actualSecret = secret || (await this.generateToken())
const hmac = crypto.createHmac('sha256', actualSecret)
hmac.update(data)
return hmac.digest('hex')
}
/**
* Verify HMAC signature
*/
async verify(data: string, signature: string, secret: string): Promise<boolean> {
const expectedSignature = await this.sign(data, secret)
return this.constantTimeCompare(signature, expectedSignature)
}
// Helper methods
private hexToBytes(hex: string): Uint8Array {
const matches = hex.match(/.{1,2}/g)
if (!matches) throw new Error('Invalid hex string')
return new Uint8Array(matches.map(byte => parseInt(byte, 16)))
}
private bytesToHex(bytes: Uint8Array | Buffer): string {
return Array.from(bytes)
.map(b => b.toString(16).padStart(2, '0'))
.join('')
}
private constantTimeCompare(a: string, b: string): boolean {
if (a.length !== b.length) return false
let result = 0
for (let i = 0; i < a.length; i++) {
result |= a.charCodeAt(i) ^ b.charCodeAt(i)
}
return result === 0
}
}

930
src/api/UniversalImportAPI.ts
View file

@ -1,930 +0,0 @@
/**
* Universal Neural Import API
*
* ALWAYS uses neural matching to map ANY data to our strict NounTypes and VerbTypes
* Never falls back to rules - neural matching is MANDATORY
*
* Handles:
* - Strings (text, JSON, CSV, YAML, Markdown)
* - Files (local paths, any format) - uses MimeTypeDetector for 2000+ types
* - URLs (web pages, APIs, documents)
* - Objects (structured data)
* - Binary data (images, PDFs via extraction)
*/
import { NounType, VerbType } from '../types/graphTypes.js'
import { Vector } from '../coreTypes.js'
import type { Brainy } from '../brainy.js'
import type { Entity, Relation } from '../types/brainy.types.js'
import { NeuralImportAugmentation } from '../neural/neuralImportAugmentation.js'
import { mimeDetector } from '../vfs/MimeTypeDetector.js'
export interface ImportSource {
type: 'string' | 'file' | 'url' | 'object' | 'binary'
data: any
format?: string // Optional hint about format
metadata?: any // Additional context
}
export interface NeuralImportResult {
entities: Array<{
id: string
type: NounType
data: any
vector: Vector
confidence: number
metadata: any
}>
relationships: Array<{
id: string
from: string
to: string
type: VerbType
weight: number
confidence: number
metadata?: any
}>
stats: {
totalProcessed: number
entitiesCreated: number
relationshipsCreated: number
averageConfidence: number
processingTimeMs: number
}
}
export interface NeuralImportProgress {
phase: 'extracting' | 'storing-entities' | 'storing-relationships' | 'complete'
message: string
current: number
total: number
entities?: number
relationships?: number
}
export class UniversalImportAPI {
private brain: Brainy<any>
private neuralImport: NeuralImportAugmentation
private embedCache = new Map<string, Vector>()
constructor(brain: Brainy<any>) {
this.brain = brain
this.neuralImport = new NeuralImportAugmentation({
confidenceThreshold: 0.0, // Accept ALL confidence levels - never reject
enableWeights: true,
skipDuplicates: false // Process everything
})
}
/**
* Initialize the neural import system
*/
async init(): Promise<void> {
// Neural import initializes itself
}
/**
* Universal import - handles ANY data source
* ALWAYS uses neural matching, NEVER falls back
*/
async import(
source: ImportSource | string | any,
options?: { onProgress?: (progress: NeuralImportProgress) => void }
): Promise<NeuralImportResult> {
const startTime = Date.now()
// Normalize source
const normalizedSource = this.normalizeSource(source)
options?.onProgress?.({
phase: 'extracting',
message: 'Extracting data from source...',
current: 0,
total: 0
})
// Extract data based on source type
const extractedData = await this.extractData(normalizedSource)
// Neural processing - MANDATORY
const neuralResults = await this.neuralProcess(extractedData)
// Store in brain
const result = await this.storeInBrain(neuralResults, options?.onProgress)
result.stats.processingTimeMs = Date.now() - startTime
options?.onProgress?.({
phase: 'complete',
message: 'Import complete',
current: result.stats.entitiesCreated + result.stats.relationshipsCreated,
total: result.stats.totalProcessed,
entities: result.stats.entitiesCreated,
relationships: result.stats.relationshipsCreated
})
return result
}
/**
* Import from URL - fetches and processes
*/
async importFromURL(url: string): Promise<NeuralImportResult> {
const response = await fetch(url)
const contentType = response.headers.get('content-type') || 'text/plain'
let data: any
if (contentType.includes('json')) {
data = await response.json()
} else if (contentType.includes('text') || contentType.includes('html')) {
data = await response.text()
} else {
// Binary data
const buffer = await response.arrayBuffer()
data = new Uint8Array(buffer)
}
return this.import({
type: 'url',
data,
format: contentType,
metadata: { url, fetchedAt: Date.now() }
})
}
/**
* Import from file - reads and processes
* Note: In browser environment, use File API instead
*
* Uses MimeTypeDetector for comprehensive format detection (2000+ types)
*/
async importFromFile(filePath: string): Promise<NeuralImportResult> {
// Read the actual file content
const { readFileSync } = await import('node:fs')
// Use MimeTypeDetector for comprehensive format detection
const mimeType = mimeDetector.detectMimeType(filePath)
const ext = filePath.split('.').pop()?.toLowerCase() || 'txt'
try {
const fileContent = readFileSync(filePath, 'utf-8')
return this.import({
type: 'file',
data: fileContent, // Actual file content
format: ext, // Keep ext for backward compatibility
metadata: {
path: filePath,
mimeType, // Add detected MIME type
importedAt: Date.now(),
fileSize: fileContent.length
}
})
} catch (error) {
throw new Error(`Failed to read file ${filePath}: ${(error as Error).message}`)
}
}
/**
* Normalize any input to ImportSource
*/
private normalizeSource(source: any): ImportSource {
// Already normalized
if (source && typeof source === 'object' && 'type' in source && 'data' in source) {
return source as ImportSource
}
// String input
if (typeof source === 'string') {
// Check if it's a URL
if (source.startsWith('http://') || source.startsWith('https://')) {
return { type: 'url', data: source }
}
// Check if it looks like a file path
if (source.includes('/') || source.includes('\\') || source.includes('.')) {
// Assume it's a file path reference
return { type: 'file', data: source }
}
// Treat as raw string data
return { type: 'string', data: source }
}
// Object/Array input
if (typeof source === 'object') {
return { type: 'object', data: source }
}
// Default to string
return { type: 'string', data: String(source) }
}
/**
* Extract structured data from source
*/
private async extractData(source: ImportSource): Promise<any[]> {
switch (source.type) {
case 'url':
// URL is in data field, need to fetch
return this.extractFromURL(source.data)
case 'file':
// File path is in data field, need to read
return this.extractFromFile(source.data)
case 'string':
return this.extractFromString(source.data, source.format)
case 'object':
return Array.isArray(source.data) ? source.data : [source.data]
case 'binary':
return this.extractFromBinary(source.data, source.format)
default:
// Unknown type, treat as object
return [source.data]
}
}
/**
* Extract data from URL
*/
private async extractFromURL(url: string): Promise<any[]> {
const result = await this.importFromURL(url)
return result.entities.map(e => e.data)
}
/**
* Extract data from file
*/
private async extractFromFile(filePath: string): Promise<any[]> {
const result = await this.importFromFile(filePath)
return result.entities.map(e => e.data)
}
/**
* Extract data from string based on format
*/
private extractFromString(data: string, format?: string): any[] {
// Try to detect format if not provided
const detectedFormat = format || this.detectFormat(data)
switch (detectedFormat) {
case 'json':
try {
const parsed = JSON.parse(data)
return Array.isArray(parsed) ? parsed : [parsed]
} catch {
// Not valid JSON, treat as text
return this.extractFromText(data)
}
case 'csv':
return this.parseCSV(data)
case 'yaml':
case 'yml':
return this.parseYAML(data)
case 'markdown':
case 'md':
return this.parseMarkdown(data)
case 'xml':
case 'html':
return this.parseHTML(data)
default:
return this.extractFromText(data)
}
}
/**
* Extract from binary data (images, PDFs, etc)
*/
private async extractFromBinary(data: Uint8Array, format?: string): Promise<any[]> {
// For now, create a single entity representing the binary data
// In production, would use OCR, image recognition, PDF extraction, etc.
return [{
type: 'binary',
format: format || 'unknown',
size: data.length,
hash: await this.hashBinary(data),
extractedAt: Date.now()
}]
}
/**
* Extract entities from plain text
*/
private extractFromText(text: string): any[] {
// Split into meaningful chunks
const chunks: any[] = []
// Split by paragraphs
const paragraphs = text.split(/\n\n+/)
for (const para of paragraphs) {
if (para.trim()) {
chunks.push({
text: para.trim(),
type: 'paragraph',
length: para.length
})
}
}
// If no paragraphs, split by sentences
if (chunks.length === 0) {
const sentences = text.match(/[^.!?]+[.!?]+/g) || [text]
for (const sentence of sentences) {
if (sentence.trim()) {
chunks.push({
text: sentence.trim(),
type: 'sentence',
length: sentence.length
})
}
}
}
return chunks
}
/**
* Neural processing - CORE of the system
* ALWAYS uses embeddings and neural matching
*/
private async neuralProcess(data: any[]): Promise<{
entities: Map<string, any>
relationships: Map<string, any>
}> {
const entities = new Map<string, any>()
const relationships = new Map<string, any>()
for (const item of data) {
// Generate embedding for the item
const embedding = await this.generateEmbedding(item)
// Determine noun type from data
const nounType = this.inferNounType(item)
const entityId = this.generateId(item)
entities.set(entityId, {
id: entityId,
type: nounType,
data: item,
vector: embedding,
confidence: 1.0,
metadata: {
...item,
_importedAt: Date.now()
}
})
// Detect relationships using neural matching
await this.detectNeuralRelationships(item, entityId, entities, relationships)
}
return { entities, relationships }
}
/**
* Generate embedding for any data
*/
private async generateEmbedding(data: any): Promise<Vector> {
// Convert to string for embedding
const text = this.dataToText(data)
// Check cache
if (this.embedCache.has(text)) {
return this.embedCache.get(text)!
}
// Generate new embedding
const embedding = await (this.brain as any).embed(text)
// Cache it
this.embedCache.set(text, embedding)
return embedding
}
/**
* Convert any data to text for embedding
*/
private dataToText(data: any): string {
if (typeof data === 'string') return data
if (typeof data === 'object') {
// Extract meaningful text from object
const parts: string[] = []
// Priority fields
const priorityFields = ['name', 'title', 'description', 'text', 'content', 'label', 'value']
for (const field of priorityFields) {
if (data[field]) {
parts.push(String(data[field]))
}
}
// Add other fields
for (const [key, value] of Object.entries(data)) {
if (!priorityFields.includes(key) && value) {
if (typeof value === 'string' || typeof value === 'number') {
parts.push(`${key}: ${value}`)
}
}
}
return parts.join(' ')
}
return JSON.stringify(data)
}
/**
* Detect relationships using neural matching
*/
private async detectNeuralRelationships(
item: any,
sourceId: string,
entities: Map<string, any>,
relationships: Map<string, any>
): Promise<void> {
if (typeof item !== 'object') return
// Look for references to other entities
for (const [key, value] of Object.entries(item)) {
// Check if this looks like a reference
if (this.looksLikeReference(key, value)) {
const targetId = String(value)
const verbType = this.inferVerbType(key)
const relationId = `${sourceId}_${verbType}_${targetId}`
relationships.set(relationId, {
id: relationId,
from: sourceId,
to: targetId,
type: verbType,
weight: 1.0,
confidence: 1.0,
metadata: {
field: key,
_importedAt: Date.now()
}
})
}
// Handle arrays of references
if (Array.isArray(value)) {
for (const arrayItem of value) {
if (this.looksLikeReference(key, arrayItem)) {
const targetId = String(arrayItem)
const verbType = this.inferVerbType(key)
const relationId = `${sourceId}_${verbType}_${targetId}`
relationships.set(relationId, {
id: relationId,
from: sourceId,
to: targetId,
type: verbType,
weight: 1.0,
confidence: 1.0,
metadata: {
field: key,
array: true,
_importedAt: Date.now()
}
})
}
}
}
}
}
/**
* Check if a field looks like a reference
*/
private looksLikeReference(key: string, value: any): boolean {
// Field name patterns that suggest references
const refPatterns = [
/[Ii]d$/, // ends with Id or id
/_id$/, // ends with _id
/^parent/i, // starts with parent
/^child/i, // starts with child
/^related/i, // starts with related
/^ref/i, // starts with ref
/^link/i, // starts with link
/^target/i, // starts with target
/^source/i, // starts with source
]
// Check if field name matches patterns
const fieldLooksLikeRef = refPatterns.some(pattern => pattern.test(key))
// Check if value looks like an ID
const valueLooksLikeId = (
typeof value === 'string' ||
typeof value === 'number'
) && String(value).length > 0
return fieldLooksLikeRef && valueLooksLikeId
}
/**
* Infer noun type from object structure using field heuristics
*/
private inferNounType(obj: any): NounType {
if (typeof obj !== 'object' || obj === null) return NounType.Thing
// Check for explicit type field
if (obj.type && typeof obj.type === 'string') {
const normalized = obj.type.charAt(0).toUpperCase() + obj.type.slice(1)
if (Object.values(NounType).includes(normalized as NounType)) {
return normalized as NounType
}
}
// Person heuristics
if (obj.email || obj.firstName || obj.lastName || obj.username || obj.age) {
return NounType.Person
}
// Organization heuristics
if (obj.companyName || obj.organizationId || obj.employees || obj.industry) {
return NounType.Organization
}
// Location heuristics
if (obj.latitude || obj.longitude || obj.address || obj.city || obj.country) {
return NounType.Location
}
// Document heuristics
if ((obj.content && (obj.title || obj.author)) || obj.documentType || obj.pages) {
return NounType.Document
}
// Event heuristics
if (obj.startTime || obj.endTime || obj.date || obj.eventType || obj.attendees) {
return NounType.Event
}
// Product heuristics
if (obj.price || obj.sku || obj.inventory || obj.productId) {
return NounType.Product
}
// Task heuristics
if ((obj.status && (obj.assignee || obj.dueDate)) || obj.priority || obj.completed !== undefined) {
return NounType.Task
}
// Dataset heuristics
if (Array.isArray(obj.data) || obj.rows || obj.columns || obj.schema) {
return NounType.Dataset
}
return NounType.Thing
}
/**
* Infer verb type from field name using common patterns
*/
private inferVerbType(fieldName: string): VerbType {
const field = fieldName.toLowerCase()
if (field.includes('parent') || field.includes('child') || field.includes('contain')) {
return VerbType.Contains
}
if (field.includes('owner') || field.includes('created') || field.includes('author')) {
return VerbType.Creates
}
if (field.includes('member') || field.includes('belong')) {
return VerbType.MemberOf
}
if (field.includes('depend') || field.includes('require')) {
return VerbType.DependsOn
}
if (field.includes('ref') || field.includes('link') || field.includes('source')) {
return VerbType.References
}
return VerbType.RelatedTo
}
/**
* Store processed data in brain
*/
private async storeInBrain(
neuralResults: {
entities: Map<string, any>
relationships: Map<string, any>
},
onProgress?: (progress: NeuralImportProgress) => void
): Promise<NeuralImportResult> {
const result: NeuralImportResult = {
entities: [],
relationships: [],
stats: {
totalProcessed: neuralResults.entities.size + neuralResults.relationships.size,
entitiesCreated: 0,
relationshipsCreated: 0,
averageConfidence: 0,
processingTimeMs: 0
}
}
let totalConfidence = 0
// Store entities
onProgress?.({
phase: 'storing-entities',
message: 'Storing entities...',
current: 0,
total: neuralResults.entities.size
})
let entitiesProcessed = 0
for (const entity of neuralResults.entities.values()) {
// Subtype precedence: extractor-set → `'extracted'` (this is the universal
// neural-extraction path, so `'extracted'` is the honest provenance label).
// Added 7.30.1 so enforcement consumers don't get rejected on auto-extraction.
const id = await this.brain.add({
data: entity.data,
type: entity.type,
subtype: (entity as any).subtype ?? 'extracted',
metadata: entity.metadata,
vector: entity.vector
})
// Update entity ID for relationship mapping
entity.id = id
result.entities.push({
...entity,
id
})
result.stats.entitiesCreated++
totalConfidence += entity.confidence
entitiesProcessed++
// Report progress periodically
if (entitiesProcessed % 10 === 0 || entitiesProcessed === neuralResults.entities.size) {
onProgress?.({
phase: 'storing-entities',
message: `Storing entities: ${entitiesProcessed}/${neuralResults.entities.size}`,
current: entitiesProcessed,
total: neuralResults.entities.size,
entities: entitiesProcessed
})
}
}
// Store relationships using batch processing
if (neuralResults.relationships.size > 0) {
onProgress?.({
phase: 'storing-relationships',
message: 'Preparing relationships...',
current: 0,
total: neuralResults.relationships.size
})
// Collect all relationship parameters. Subtype `extracted` matches the
// entity-side label so consumers can query "everything from this neural pass"
// via `(type, subtype: 'extracted')` (added 7.30.1).
const relationshipParams: Array<{from: string; to: string; type: VerbType; subtype?: string; weight?: number; metadata?: any}> = []
for (const relation of neuralResults.relationships.values()) {
// Map to actual entity IDs
const sourceEntity = Array.from(neuralResults.entities.values())
.find(e => e.id === relation.from)
const targetEntity = Array.from(neuralResults.entities.values())
.find(e => e.id === relation.to)
if (sourceEntity && targetEntity) {
relationshipParams.push({
from: sourceEntity.id,
to: targetEntity.id,
type: relation.type,
subtype: (relation as any).subtype ?? 'extracted',
weight: relation.weight,
metadata: relation.metadata
})
totalConfidence += relation.confidence
}
}
// Batch create relationships with progress
if (relationshipParams.length > 0) {
const relationshipIds = await this.brain.relateMany({
items: relationshipParams,
parallel: true,
chunkSize: 100,
continueOnError: true,
onProgress: (done, total) => {
onProgress?.({
phase: 'storing-relationships',
message: `Building relationships: ${done}/${total}`,
current: done,
total: total,
entities: result.stats.entitiesCreated,
relationships: done
})
}
})
// Map results back
relationshipIds.forEach((id, index) => {
if (id && relationshipParams[index]) {
result.relationships.push({
id,
from: relationshipParams[index].from,
to: relationshipParams[index].to,
type: relationshipParams[index].type,
weight: relationshipParams[index].weight || 1,
confidence: 0.5, // Default confidence
metadata: relationshipParams[index].metadata
})
}
})
result.stats.relationshipsCreated = relationshipIds.length
}
}
// Calculate average confidence
const totalItems = result.stats.entitiesCreated + result.stats.relationshipsCreated
result.stats.averageConfidence = totalItems > 0 ? totalConfidence / totalItems : 0
return result
}
// Helper methods for parsing different formats
private detectFormat(data: string): string {
const trimmed = data.trim()
// JSON
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) ||
(trimmed.startsWith('[') && trimmed.endsWith(']'))) {
return 'json'
}
// CSV (has commas and newlines)
if (trimmed.includes(',') && trimmed.includes('\n')) {
return 'csv'
}
// YAML (has colons and indentation)
if (trimmed.includes(':') && (trimmed.includes('\n ') || trimmed.includes('\n\t'))) {
return 'yaml'
}
// Markdown (has headers)
if (trimmed.includes('#') || trimmed.includes('```')) {
return 'markdown'
}
// HTML/XML
if (trimmed.includes('<') && trimmed.includes('>')) {
return trimmed.toLowerCase().includes('<!doctype html') ? 'html' : 'xml'
}
return 'text'
}
private parseCSV(data: string): any[] {
// Reuse the CSV parser from neural import
const lines = data.split('\n').filter(l => l.trim())
if (lines.length === 0) return []
const headers = lines[0].split(',').map(h => h.trim())
const results = []
for (let i = 1; i < lines.length; i++) {
const values = lines[i].split(',').map(v => v.trim())
const obj: any = {}
headers.forEach((header, index) => {
obj[header] = values[index] || ''
})
results.push(obj)
}
return results
}
private parseYAML(data: string): any[] {
// Simple YAML parser
const results = []
const lines = data.split('\n')
let current: any = null
for (const line of lines) {
const trimmed = line.trim()
if (!trimmed || trimmed.startsWith('#')) continue
if (trimmed.startsWith('- ')) {
// Array item
const value = trimmed.substring(2)
if (!current) {
results.push(value)
} else {
if (!current._items) current._items = []
current._items.push(value)
}
} else if (trimmed.includes(':')) {
// Key-value
const [key, ...valueParts] = trimmed.split(':')
const value = valueParts.join(':').trim()
if (!current) {
current = {}
results.push(current)
}
current[key.trim()] = value
}
}
return results.length > 0 ? results : [{ text: data }]
}
private parseMarkdown(data: string): any[] {
const results = []
const lines = data.split('\n')
let current: any = null
let inCodeBlock = false
for (const line of lines) {
if (line.startsWith('```')) {
inCodeBlock = !inCodeBlock
if (inCodeBlock && current) {
current.code = ''
}
continue
}
if (inCodeBlock && current) {
current.code += line + '\n'
} else if (line.startsWith('#')) {
// Header
const level = line.match(/^#+/)?.[0].length || 1
const text = line.replace(/^#+\s*/, '')
current = {
type: 'heading',
level,
text
}
results.push(current)
} else if (line.trim()) {
// Paragraph
if (!current || current.type !== 'paragraph') {
current = {
type: 'paragraph',
text: ''
}
results.push(current)
}
current.text += line + ' '
}
}
return results
}
private parseHTML(data: string): any[] {
// Simple HTML text extraction
const text = data
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '') // Remove scripts
.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '') // Remove styles
.replace(/<[^>]+>/g, ' ') // Remove tags
.replace(/\s+/g, ' ') // Normalize whitespace
.trim()
return this.extractFromText(text)
}
private generateId(data: any): string {
// Generate deterministic ID based on content
const text = this.dataToText(data)
const hash = this.simpleHash(text)
return `import_${hash}_${Date.now()}`
}
private simpleHash(text: string): string {
let hash = 0
for (let i = 0; i < text.length; i++) {
const char = text.charCodeAt(i)
hash = ((hash << 5) - hash) + char
hash = hash & hash
}
return Math.abs(hash).toString(36)
}
private async hashBinary(data: Uint8Array): Promise<string> {
// Simple binary hash
let hash = 0
for (let i = 0; i < Math.min(data.length, 1000); i++) {
hash = ((hash << 5) - hash) + data[i]
hash = hash & hash
}
return Math.abs(hash).toString(36)
}
}

513
src/brainy.ts
View file

File diff suppressed because it is too large Load diff

99
src/cli/commands/core.ts
View file

@ -7,10 +7,11 @@
*/
import chalk from 'chalk'
import ora from 'ora'
import ora, { type Ora } from 'ora'
import inquirer from 'inquirer'
import { Brainy } from '../../brainy.js'
import { BrainyTypes, NounType, VerbType } from '../../index.js'
import type { Entity, Result } from '../../types/brainy.types.js'
interface CoreOptions {
verbose?: boolean
@ -87,7 +88,7 @@ export const coreCommands = {
* Add data to the neural database
*/
async add(text: string | undefined, options: AddOptions) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no text provided
if (!text) {
@ -222,7 +223,7 @@ export const coreCommands = {
* Search the neural database with Triple Intelligence™
*/
async search(query: string | undefined, options: SearchOptions) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no query provided
if (!query) {
@ -407,8 +408,13 @@ export const coreCommands = {
if (result.score !== undefined) {
console.log(chalk.green(` Score: ${(result.score * 100).toFixed(1)}%`))
if (options.explain && (result as any).scores) {
const scores = (result as any).scores
// Per-intelligence sub-scores aren't part of the public Result
// contract (the typed breakdown lives on `explanation`) — read
// defensively so --explain degrades to no breakdown when absent.
const scores = (result as Result & {
scores?: { vector?: number; graph?: number; field?: number }
}).scores
if (options.explain && scores) {
if (scores.vector !== undefined) {
console.log(chalk.dim(` Vector: ${(scores.vector * 100).toFixed(1)}%`))
}
@ -422,24 +428,28 @@ export const coreCommands = {
}
// Show type
if ((entity as any).type) {
console.log(chalk.dim(` Type: ${(entity as any).type}`))
if (entity.type) {
console.log(chalk.dim(` Type: ${entity.type}`))
}
// Show content preview
if ((entity as any).content) {
const preview = (entity as any).content.substring(0, 80)
console.log(chalk.dim(` Content: ${preview}${(entity as any).content.length > 80 ? '...' : ''}`))
// Show content preview. `content` is not a standard Entity field —
// surfaced defensively for records that carry flattened text content.
const content = (entity as Entity & { content?: string }).content
if (content) {
const preview = content.substring(0, 80)
console.log(chalk.dim(` Content: ${preview}${content.length > 80 ? '...' : ''}`))
}
// Show metadata
if ((entity as any).metadata && Object.keys((entity as any).metadata).length > 0) {
console.log(chalk.dim(` Metadata: ${JSON.stringify((entity as any).metadata)}`))
if (entity.metadata && Object.keys(entity.metadata).length > 0) {
console.log(chalk.dim(` Metadata: ${JSON.stringify(entity.metadata)}`))
}
// Show relationships
if (options.includeRelations && (result as any).relations) {
const relations = (result as any).relations
// Show relationships. `relations` isn't part of the public Result
// contract — read defensively so --include-relations degrades to
// no output when find() doesn't attach it.
const relations = (result as Result & { relations?: unknown[] }).relations
if (options.includeRelations && relations) {
if (relations.length > 0) {
console.log(chalk.dim(` Relations: ${relations.length} connections`))
}
@ -487,7 +497,7 @@ export const coreCommands = {
* Get item by ID
*/
async get(id: string | undefined, options: GetOptions) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no ID provided
if (!id) {
@ -527,18 +537,25 @@ export const coreCommands = {
if (!options.json) {
console.log(chalk.cyan('\nItem Details:'))
console.log(` ID: ${item.id}`)
console.log(` Content: ${(item as any).content || 'N/A'}`)
// `content` is not a standard Entity field — see the search() note.
console.log(` Content: ${(item as Entity & { content?: string }).content || 'N/A'}`)
if (item.metadata) {
console.log(` Metadata: ${JSON.stringify(item.metadata, null, 2)}`)
}
if (options.withConnections) {
// Get verbs/relationships
// Get connections if method exists
const connections = (brain as any).getConnections ? await (brain as any).getConnections(id) : []
// Brainy's public API has no getConnections() — this optional probe
// predates related() and resolves to an empty list on current
// builds. Typed to match the legacy edge shape it rendered.
const legacyBrain = brain as Brainy & {
getConnections?: (id: string | undefined) => Promise<
Array<{ source: string; type: string; target: string }>
>
}
const connections = legacyBrain.getConnections ? await legacyBrain.getConnections(id) : []
if (connections && connections.length > 0) {
console.log(chalk.cyan('\nConnections:'))
connections.forEach((conn: any) => {
connections.forEach((conn) => {
console.log(` ${conn.source} --[${conn.type}]--> ${conn.target}`)
})
}
@ -561,7 +578,7 @@ export const coreCommands = {
* Create relationship between items
*/
async relate(source: string | undefined, verb: string | undefined, target: string | undefined, options: RelateOptions) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if parameters missing
if (!source || !verb || !target) {
@ -630,10 +647,12 @@ export const coreCommands = {
// `--subtype <value>` → Brainy default `'cli-relate'`. Same rationale
// as `brainy add` — guarantees CLI works under strict-mode brains
// (added 7.30.1).
// The verb arrives as a raw CLI string; relate() validates it against
// the VerbType vocabulary at runtime.
const result = await brain.relate({
from: source,
to: target,
type: verb as any,
type: verb as VerbType,
subtype: options.subtype ?? 'cli-relate',
metadata
})
@ -664,7 +683,7 @@ export const coreCommands = {
* Update an existing entity
*/
async update(id: string | undefined, options: AddOptions & { content?: string }) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no ID provided
if (!id) {
@ -770,10 +789,10 @@ export const coreCommands = {
},
/**
* Delete an entity
* Remove an entity
*/
async deleteEntity(id: string | undefined, options: CoreOptions & { force?: boolean }) {
let spinner: any = null
async removeEntity(id: string | undefined, options: CoreOptions & { force?: boolean }) {
let spinner: Ora | null = null
try {
// Interactive mode if no ID provided
if (!id) {
@ -781,7 +800,7 @@ export const coreCommands = {
{
type: 'input',
name: 'id',
message: 'Entity ID to delete:',
message: 'Entity ID to remove:',
validate: (input: string) => input.trim().length > 0 || 'ID cannot be empty'
},
{
@ -793,7 +812,7 @@ export const coreCommands = {
])
if (!answers.confirm) {
console.log(chalk.yellow('Delete cancelled'))
console.log(chalk.yellow('Operation cancelled'))
return
}
@ -803,36 +822,36 @@ export const coreCommands = {
const answer = await inquirer.prompt([{
type: 'confirm',
name: 'confirm',
message: `Delete entity ${id}? This cannot be undone.`,
message: `Remove entity ${id}? This cannot be undone.`,
default: false
}])
if (!answer.confirm) {
console.log(chalk.yellow('Delete cancelled'))
console.log(chalk.yellow('Operation cancelled'))
return
}
}
spinner = ora('Deleting entity...').start()
spinner = ora('Removing entity...').start()
const brain = getBrainy()
await brain.init()
await brain.delete(id)
await brain.remove(id)
spinner.succeed('Entity deleted successfully')
spinner.succeed('Entity removed successfully')
if (!options.json) {
console.log(chalk.green(`✓ Deleted entity: ${id}`))
console.log(chalk.green(`✓ Removed entity: ${id}`))
} else {
formatOutput({ id, deleted: true }, options)
formatOutput({ id, removed: true }, options)
}
// One-shot command — see add() for why the explicit close + exit.
await brain.close()
process.exit(0)
} catch (error: any) {
if (spinner) spinner.fail('Failed to delete entity')
console.error(chalk.red('Delete failed:', error.message))
if (spinner) spinner.fail('Failed to remove entity')
console.error(chalk.red('Remove failed:', error.message))
process.exit(1)
}
},
@ -841,7 +860,7 @@ export const coreCommands = {
* Remove a relationship
*/
async unrelate(id: string | undefined, options: CoreOptions & { force?: boolean }) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no ID provided
if (!id) {

2
src/cli/commands/data.ts
View file

@ -10,7 +10,7 @@
* The legacy backup/import/export facade that used to live behind this
* command was superseded in 8.0: snapshots are `brainy snapshot` / `brainy
* restore` (the Db API's `persist()`/`restore()`), and data ingestion is
* `brainy import` (UniversalImportAPI).
* `brainy import` (`brain.import()`).
*/
import chalk from 'chalk'

23
src/cli/commands/import.ts
View file

@ -3,7 +3,7 @@
* @description Import commands — neural import and data import.
*
* Complete import system exposing ALL Brainy import capabilities:
* - UniversalImportAPI: Neural import with AI type matching
* - `brain.import()`: universal ingestion with neural type matching
* - DirectoryImporter: VFS directory imports
*
* Supports: files, directories, URLs, all formats. Backup/restore is the
@ -11,19 +11,23 @@
*/
import chalk from 'chalk'
import ora from 'ora'
import ora, { type Ora } from 'ora'
import inquirer from 'inquirer'
import { readFileSync, statSync, existsSync } from 'node:fs'
import { Brainy } from '../../brainy.js'
import { NounType } from '../../types/graphTypes.js'
import type { SupportedFormat } from '../../import/FormatDetector.js'
interface ImportOptions {
verbose?: boolean
json?: boolean
pretty?: boolean
quiet?: boolean
// Format options
format?: 'json' | 'csv' | 'jsonl' | 'yaml' | 'markdown' | 'html' | 'xml' | 'text'
// Format options. Populated by commander straight from `--format` with no
// validation, so the runtime value is whatever the user typed — kept as a
// plain string and narrowed to brain.import()'s SupportedFormat at the call
// boundary (the import extractor rejects unsupported values at runtime).
format?: string
// Import behavior
recursive?: boolean
batchSize?: string
@ -60,11 +64,11 @@ const formatOutput = (data: any, options: ImportOptions): void => {
export const importCommands = {
/**
* Enhanced import using UniversalImportAPI
* Supports files, directories, URLs, all formats
* Enhanced import via `brain.import()`.
* Supports files, directories, URLs, all formats.
*/
async import(source: string | undefined, options: ImportOptions) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no source provided
if (!source) {
@ -260,7 +264,8 @@ export const importCommands = {
} else {
// File import with progress
result = await brain.import(source, {
format: options.format as any,
// Raw CLI string → SupportedFormat (see ImportOptions.format note).
format: options.format as SupportedFormat | undefined,
enableNeuralExtraction: options.extractEntities !== false,
enableRelationshipInference: options.detectRelationships !== false,
enableConceptExtraction: options.extractConcepts || false,
@ -422,7 +427,7 @@ export const importCommands = {
* VFS-specific import (files/directories into VFS)
*/
async vfsImport(source: string | undefined, options: ImportOptions) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no source provided
if (!source) {

6
src/cli/commands/insights.ts
View file

@ -5,7 +5,7 @@
*/
import chalk from 'chalk'
import ora from 'ora'
import ora, { type Ora } from 'ora'
import inquirer from 'inquirer'
import Table from 'cli-table3'
import { Brainy } from '../../brainy.js'
@ -192,7 +192,7 @@ export const insightsCommands = {
* Get field values for a specific field
*/
async fieldValues(field: string | undefined, options: InsightsOptions & { limit?: string }) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no field provided
if (!field) {
@ -274,7 +274,7 @@ export const insightsCommands = {
* Get optimal query plan for filters
*/
async queryPlan(options: InsightsOptions & { filters?: string }) {
let spinner: any = null
let spinner: Ora | null = null
try {
let filters: Record<string, any> = {}

24
src/cli/commands/inspect.ts
View file

@ -15,6 +15,8 @@
import chalk from 'chalk'
import ora from 'ora'
import { Brainy } from '../../brainy.js'
import type { RelatedParams } from '../../types/brainy.types.js'
import type { NounType, VerbType } from '../../types/graphTypes.js'
interface InspectOptions {
fresh?: boolean
@ -109,8 +111,10 @@ export const inspectCommands = {
const where = options.where ? JSON.parse(options.where) : undefined
const limit = options.limit ? parseInt(options.limit, 10) : 20
const offset = options.offset ? parseInt(options.offset, 10) : 0
// --type arrives as a raw CLI string; find() resolves it against the
// NounType vocabulary at runtime.
const results = await brain.find({
type: options.type as any,
type: options.type as NounType | undefined,
where,
limit,
offset
@ -158,12 +162,16 @@ export const inspectCommands = {
if (spinner) spinner.text = 'Walking graph…'
const direction = options.direction ?? 'both'
const limit = options.limit ? parseInt(options.limit, 10) : 50
const rels = await brain.getRelations({
// `direction` is accepted by the CLI surface but RelatedParams has
// no such filter — related() reads from/to/type/limit and ignores
// extras, so the assertion only admits the extra key, it doesn't change
// what the query does. --type arrives as a raw CLI string.
const rels = await brain.related({
from: id,
direction,
type: options.type as any,
type: options.type as VerbType | undefined,
limit
} as any)
} as RelatedParams)
spinner?.succeed(`Found ${rels.length} relationship(s)`)
emit(rels, options)
await brain.close()
@ -185,7 +193,7 @@ export const inspectCommands = {
const brain = await openReader(path, options)
if (spinner) spinner.text = 'Planning query…'
const where = options.where ? JSON.parse(options.where) : {}
const plan = await brain.explain({ type: options.type as any, where })
const plan = await brain.explain({ type: options.type as NounType | undefined, where })
spinner?.succeed('Plan ready')
emit(plan, options)
await brain.close()
@ -228,7 +236,7 @@ export const inspectCommands = {
// dedicated random-sample API in Brainy core for this v1.
const window = Math.min(Math.max(n * 10, 50), 1000)
const pool = await brain.find({
type: options.type as any,
type: options.type as NounType | undefined,
limit: window
})
const sample = pool
@ -275,7 +283,7 @@ export const inspectCommands = {
let offset = 0
while (true) {
const page = await brain.find({
type: options.type as any,
type: options.type as NounType | undefined,
limit: batch,
offset
})
@ -310,7 +318,7 @@ export const inspectCommands = {
try {
const brain = await openReader(path, { ...options, quiet: true })
const page = await brain.find({
type: options.type as any,
type: options.type as NounType | undefined,
limit: 200
})
for (const e of page) {

9
src/cli/commands/neural.ts
View file

@ -10,10 +10,11 @@
import inquirer from 'inquirer'
import chalk from 'chalk'
import ora from 'ora'
import ora, { type Ora } from 'ora'
import fs from 'node:fs'
import path from 'node:path'
import { Brainy } from '../../brainy.js'
import type { NeuralClusterParams } from '../../types/brainy.types.js'
interface CommandArguments {
action?: string;
@ -113,10 +114,12 @@ async function handleSimilarCommand(neural: any, argv: CommandArguments): Promis
}
async function handleClustersCommand(neural: any, argv: CommandArguments): Promise<void> {
let spinner: any = null
let spinner: Ora | null = null
try {
let options: any = {
algorithm: argv.algorithm as any,
// --algorithm arrives as a raw CLI string; the interactive prompt below
// and neural.clusters() constrain it to the supported algorithms.
algorithm: argv.algorithm as NeuralClusterParams['algorithm'],
threshold: argv.threshold,
maxClusters: argv.limit
}

8
src/cli/commands/nlp.ts
View file

@ -5,7 +5,7 @@
*/
import chalk from 'chalk'
import ora from 'ora'
import ora, { type Ora } from 'ora'
import inquirer from 'inquirer'
import Table from 'cli-table3'
import { Brainy } from '../../brainy.js'
@ -37,7 +37,7 @@ export const nlpCommands = {
* Extract entities from text
*/
async extract(text: string | undefined, options: NLPOptions) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no text provided
if (!text) {
@ -126,7 +126,7 @@ export const nlpCommands = {
* Extract concepts from text
*/
async extractConcepts(text: string | undefined, options: NLPOptions & { threshold?: string }) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no text provided
if (!text) {
@ -197,7 +197,7 @@ export const nlpCommands = {
* Analyze text with full NLP pipeline
*/
async analyze(text: string | undefined, options: NLPOptions) {
let spinner: any = null
let spinner: Ora | null = null
try {
// Interactive mode if no text provided
if (!text) {

6
src/cli/commands/storage.ts
View file

@ -6,7 +6,7 @@
* - `storage status` — backend, root directory, entity/relationship counts,
* writer-lock holder, and operational mode (rendered from `brain.stats()`).
* - `storage batch-delete <file>` — delete a newline-separated list of entity
* IDs through the public `brain.delete()` path (vectors, metadata, indexes,
* IDs through the public `brain.remove()` path (vectors, metadata, indexes,
* and statistics all stay consistent), with per-ID retry and an optional
* continue-on-error mode.
*
@ -144,7 +144,7 @@ export const storageCommands = {
/**
* @description Delete a list of entity IDs (one per line in a file) through
* the public `brain.delete()` path so vectors, metadata, graph edges,
* the public `brain.remove()` path so vectors, metadata, graph edges,
* indexes, and statistics all stay consistent. Asks for confirmation before
* deleting; failed IDs are retried up to `--max-retries` times, and
* `--continue-on-error` keeps going past IDs that still fail.
@ -204,7 +204,7 @@ export const storageCommands = {
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
await brain.delete(id)
await brain.remove(id)
succeeded = true
break
} catch (error) {

18
src/cli/commands/utility.ts
View file

@ -30,6 +30,15 @@ interface BenchmarkOptions extends UtilityOptions {
iterations?: string
}
/** Per-operation timing summary produced by `brainy benchmark`. */
interface BenchmarkOperationStats {
avg: string
min: number
max: number
median: number
ops: string
}
let brainyInstance: Brainy | null = null
const getBrainy = (): Brainy => {
@ -188,7 +197,10 @@ export const utilityCommands = {
console.log(chalk.cyan(`\n🚀 Running Benchmarks (${iterations} iterations)\n`))
const results: any = {
const results: {
operations: Record<string, BenchmarkOperationStats>
summary: { totalOperations?: number; averageOpsPerSec?: string }
} = {
operations: {},
summary: {}
}
@ -253,7 +265,7 @@ export const utilityCommands = {
}
// Calculate summary
const totalOps: number = (Object.values(results.operations) as any[]).reduce((sum: number, op: any) =>
const totalOps: number = Object.values(results.operations).reduce((sum: number, op) =>
sum + parseFloat(op.ops), 0)
results.summary = {
@ -277,7 +289,7 @@ export const utilityCommands = {
style: { head: [], border: [] }
})
Object.entries(results.operations).forEach(([op, stats]: [string, any]) => {
Object.entries(results.operations).forEach(([op, stats]) => {
table.push([
op,
stats.avg,

29
src/cli/commands/vfs.ts
View file

@ -9,6 +9,7 @@ import ora from 'ora'
import Table from 'cli-table3'
import { readFileSync, writeFileSync } from 'node:fs'
import { Brainy } from '../../brainy.js'
import type { SearchResult, VFSDirent } from '../../vfs/types.js'
interface VFSOptions {
verbose?: boolean
@ -54,8 +55,9 @@ export const vfsCommands = {
try {
const brain = await getBrainy() // Await async getBrainy
// VFS auto-initialized, no need for vfs.init()
// --encoding arrives as a raw CLI string (Node validates it at use time).
const buffer = await brain.vfs.readFile(path, {
encoding: options.encoding as any
encoding: options.encoding as BufferEncoding | undefined
})
spinner.succeed('File read successfully')
@ -103,8 +105,9 @@ export const vfsCommands = {
process.exit(1)
}
// --encoding arrives as a raw CLI string (Node validates it at use time).
await brain.vfs.writeFile(path, data, {
encoding: options.encoding as any
encoding: options.encoding as BufferEncoding | undefined
})
spinner.succeed('File written successfully')
@ -135,7 +138,9 @@ export const vfsCommands = {
try {
const brain = await getBrainy()
const entries = await brain.vfs.readdir(path, { withFileTypes: true })
// withFileTypes: true selects the VFSDirent[] branch of readdir's
// `string[] | VFSDirent[]` union return type.
const entries = await brain.vfs.readdir(path, { withFileTypes: true }) as VFSDirent[]
spinner.succeed(`Found ${Array.isArray(entries) ? entries.length : 0} items`)
@ -154,13 +159,14 @@ export const vfsCommands = {
style: { head: [], border: [] }
})
for (const entry of entries as any[]) {
for (const entry of entries) {
if (!options.all && entry.name.startsWith('.')) continue
const isDirectory = entry.type === 'directory'
const stat = await brain.vfs.stat(`${path}/${entry.name}`)
table.push([
entry.isDirectory() ? chalk.blue('DIR') : 'FILE',
entry.isDirectory() ? '-' : formatBytes(stat.size),
isDirectory ? chalk.blue('DIR') : 'FILE',
isDirectory ? '-' : formatBytes(stat.size),
formatDate(stat.mtime),
entry.name
])
@ -170,10 +176,10 @@ export const vfsCommands = {
} else {
// Simple format
console.log()
for (const entry of entries as any[]) {
for (const entry of entries) {
if (!options.all && entry.name.startsWith('.')) continue
if (entry.isDirectory()) {
if (entry.type === 'directory') {
console.log(chalk.blue(entry.name + '/'))
} else {
console.log(entry.name)
@ -322,8 +328,11 @@ export const vfsCommands = {
if (result.score) {
console.log(chalk.dim(` Score: ${(result.score * 100).toFixed(1)}%`))
}
if ((result as any).excerpt) {
console.log(chalk.dim(` ${(result as any).excerpt}`))
// `excerpt` isn't part of the VFS SearchResult contract — search()
// doesn't attach it today; read defensively.
const excerpt = (result as SearchResult & { excerpt?: string }).excerpt
if (excerpt) {
console.log(chalk.dim(` ${excerpt}`))
}
console.log()
})

10
src/cli/index.ts
View file

@ -46,7 +46,7 @@ ${chalk.cyan('Examples:')}
$ brainy add "React is a JavaScript library"
$ brainy find "JavaScript frameworks"
$ brainy update <id> --content "Updated content"
$ brainy delete <id> ${chalk.dim('# Requires confirmation')}
$ brainy remove <id> ${chalk.dim('# Requires confirmation')}
$ brainy search "react" --type Component --where '{"tested":true}'
${chalk.dim('# Neural API')}
@ -143,10 +143,10 @@ program
.action(coreCommands.update)
program
.command('delete [id]')
.description('Delete an entity (interactive if no ID, requires confirmation)')
.command('remove [id]')
.description('Remove an entity (interactive if no ID, requires confirmation)')
.option('-f, --force', 'Skip confirmation prompt')
.action(coreCommands.deleteEntity)
.action(coreCommands.removeEntity)
program
.command('unrelate [id]')
@ -472,7 +472,7 @@ program
.addCommand(
new Command('batch-delete')
.argument('<file>', 'File containing entity IDs (one per line)')
.description('Delete a list of entities through the public delete path (with retry)')
.description('Delete a list of entities through the public remove() path (with retry)')
.option('--max-retries <n>', 'Maximum retry attempts per ID', '3')
.option('--continue-on-error', 'Continue past IDs that still fail after retries')
.action((file, options) => {

11
src/config/extensibleConfig.ts
View file

@ -126,7 +126,10 @@ class ConfigurationRegistry {
if (await provider.detect()) {
const config = await provider.getConfig()
return {
type: provider.type as any,
// Registered providers deliberately extend the built-in storage
// vocabulary ('redis', 'mongodb', ...) — the registry contract is
// open, so narrow the provider's free-form type string here.
type: provider.type as StorageConfigResult['type'],
config,
reason: `Auto-detected ${provider.name}`,
autoSelected: true
@ -313,7 +316,11 @@ export function registerPresetAugmentation(name: string, config: PresetConfig):
* Example preset for Redis-based caching service
*/
export const redisCachePreset: PresetConfig = {
storage: 'redis' as any, // Extended storage type
// 'redis' is an extended storage type outside the built-in StorageOption
// enum — extension presets resolve it at runtime through the provider
// registry, so this is a genuine typed boundary (hence the double
// assertion: the literal has no overlap with the enum).
storage: 'redis' as unknown as StorageOption,
model: ModelPrecision.Q8,
features: ['core', 'cache', 'search'],
distributed: true,

16
src/config/zeroConfig.ts
View file

@ -149,7 +149,9 @@ export async function processZeroConfig(input?: string | BrainyZeroConfig): Prom
// Handle string shorthand (preset name)
if (typeof input === 'string') {
if (input in PRESETS) {
config = { mode: input as any }
// The `in` guard above proved the string is a preset key, which is
// exactly the BrainyZeroConfig mode union.
config = { mode: input as keyof typeof PRESETS }
} else {
throw new Error(`Unknown preset: ${input}. Valid presets: ${Object.keys(PRESETS).join(', ')}`)
}
@ -263,7 +265,17 @@ export async function processZeroConfig(input?: string | BrainyZeroConfig): Prom
// Apply distributed preset settings if applicable
if (config.mode === 'writer' || config.mode === 'reader') {
const presetSettings = PRESETS[config.mode] as any // Cast to any since we know these presets have additional properties
// The writer/reader presets each carry their own subset of the
// distributed flags beyond the base preset shape — widen to the union of
// those flags so both branches read uniformly.
const presetSettings: {
distributed: boolean
role: 'writer' | 'reader'
readOnly?: boolean
writeOnly?: boolean
allowDirectReads?: boolean
lazyLoadInReadOnlyMode?: boolean
} = PRESETS[config.mode]
// Apply distributed-specific settings
finalConfig.distributed = presetSettings.distributed

4
src/coreTypes.ts
View file

@ -389,7 +389,7 @@ export interface HNSWVerbWithMetadata {
// `ReportsTo` relationship might have subtype 'direct' vs 'dotted-line'; a `RelatedTo`
// edge might carry 'spouse' / 'sibling' / 'colleague'). Flat string (no hierarchy) —
// consumers decide the vocabulary. Indexed and rolled up into per-VerbType statistics
// so it's queryable (`getRelations({ verb, subtype })`) and aggregable
// so it's queryable (`related({ verb, subtype })`) and aggregable
// (`groupBy:['subtype']`) on the standard-field fast path, never falling through to
// the metadata fallback.
subtype?: string
@ -417,7 +417,7 @@ export interface HNSWVerbWithMetadata {
* Verb representing a relationship between nouns
* Stored separately from the HNSW vector index for lightweight performance.
* This is the canonical verb shape — Brainy's internal graph engine, the public
* `relate()` / `getRelations()` surface, and storage adapters all speak it.
* `relate()` / `related()` surface, and storage adapters all speak it.
*/
export interface GraphVerb {
id: string // Unique identifier — always a UUID (8.0 contract: brainy generates every verb id, so providers may key verb-int interning on the raw UUID bytes)

140
src/critical/model-guardian.ts
View file

@ -1,140 +0,0 @@
/**
* MODEL GUARDIAN - CRITICAL PATH
*
* THIS IS THE MOST CRITICAL COMPONENT OF BRAINY
* Without the exact model, users CANNOT access their data
*
* Requirements:
* 1. Model MUST be all-MiniLM-L6-v2-q8 (bundled in package)
* 2. Model MUST be available at runtime (embedded in npm package)
* 3. Model MUST produce consistent 384-dim embeddings
* 4. System MUST fail fast if model unavailable in production
*/
import { WASMEmbeddingEngine } from '../embeddings/wasm/index.js'
// CRITICAL: These values MUST NEVER CHANGE
const CRITICAL_MODEL_CONFIG = {
modelName: 'all-MiniLM-L6-v2-q8',
embeddingDimensions: 384,
// Model is bundled in package - no external downloads needed
bundled: true
}
export class ModelGuardian {
private static instance: ModelGuardian
private isVerified = false
private lastVerification: Date | null = null
private constructor() {
// Model is bundled - no path detection needed
}
static getInstance(): ModelGuardian {
if (!ModelGuardian.instance) {
ModelGuardian.instance = new ModelGuardian()
}
return ModelGuardian.instance
}
/**
* CRITICAL: Verify model availability and integrity
* This MUST be called before any embedding operations
*/
async ensureCriticalModel(): Promise<void> {
// Check if already verified in this session
if (this.isVerified && this.lastVerification) {
const hoursSinceVerification =
(Date.now() - this.lastVerification.getTime()) / (1000 * 60 * 60)
if (hoursSinceVerification < 24) {
return
}
}
// Verify the bundled WASM model works
const modelWorks = await this.verifyBundledModel()
if (modelWorks) {
this.isVerified = true
this.lastVerification = new Date()
return
}
// CRITICAL FAILURE
throw new Error(
'🚨 CRITICAL FAILURE: Bundled transformer model not working!\n' +
'The model is REQUIRED for Brainy to function.\n' +
'Users CANNOT access their data without it.\n' +
'This indicates a package installation issue.'
)
}
/**
* Verify the bundled WASM model works correctly
*/
private async verifyBundledModel(): Promise<boolean> {
try {
const engine = WASMEmbeddingEngine.getInstance()
// Initialize the engine (loads bundled model)
await engine.initialize()
// Test embedding generation
const testEmbedding = await engine.embed('test verification')
// Verify dimensions
if (testEmbedding.length !== CRITICAL_MODEL_CONFIG.embeddingDimensions) {
console.error(
`❌ CRITICAL: Model dimension mismatch!\n` +
`Expected: ${CRITICAL_MODEL_CONFIG.embeddingDimensions}\n` +
`Got: ${testEmbedding.length}`
)
return false
}
// Verify normalization (should be unit length)
const norm = Math.sqrt(testEmbedding.reduce((sum, v) => sum + v * v, 0))
if (Math.abs(norm - 1.0) > 0.01) {
console.error(`❌ CRITICAL: Embeddings not normalized! Norm: ${norm}`)
return false
}
return true
} catch (error) {
console.error('❌ Model verification failed:', error)
return false
}
}
/**
* Get model status for diagnostics
*/
async getStatus(): Promise<{
verified: boolean
lastVerification: Date | null
modelName: string
dimensions: number
bundled: boolean
}> {
return {
verified: this.isVerified,
lastVerification: this.lastVerification,
modelName: CRITICAL_MODEL_CONFIG.modelName,
dimensions: CRITICAL_MODEL_CONFIG.embeddingDimensions,
bundled: CRITICAL_MODEL_CONFIG.bundled
}
}
/**
* Force re-verification (for testing)
*/
async forceReverify(): Promise<void> {
this.isVerified = false
this.lastVerification = null
await this.ensureCriticalModel()
}
}
// Export singleton instance
export const modelGuardian = ModelGuardian.getInstance()

193
src/data/expandedKeywordDictionary.ts
View file

@ -1,193 +0,0 @@
/**
* Expanded Keyword Dictionary for Semantic Type Inference
*
* Comprehensive keyword-to-type mappings including:
* - Canonical keywords (primary terms)
* - Synonyms (alternative terms with slightly lower confidence)
* - Domain-specific variations
* - Common abbreviations
*
* Expanded from 767 → 1500+ keywords for better semantic coverage
*/
import { NounType } from '../types/graphTypes.js'
export interface KeywordDefinition {
keyword: string
type: NounType
confidence: number // 0.7-0.95 (higher = more canonical)
isCanonical: boolean // True for primary terms, false for synonyms
}
/**
* Expanded keyword dictionary (1500+ keywords for 31 NounTypes)
*/
export const EXPANDED_KEYWORD_DICTIONARY: KeywordDefinition[] = [
// ========== Person - Medical Professions ==========
// Canonical
{ keyword: 'doctor', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'physician', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'surgeon', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'nurse', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'cardiologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'oncologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'neurologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'psychiatrist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'psychologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'radiologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'pathologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'anesthesiologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'dermatologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'pediatrician', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'obstetrician', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'gynecologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'ophthalmologist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'dentist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'orthodontist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'pharmacist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'paramedic', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'therapist', type: NounType.Person, confidence: 0.90, isCanonical: true },
// Synonyms
{ keyword: 'medic', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'practitioner', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'clinician', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'medical professional', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'healthcare worker', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'medical doctor', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'registered nurse', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'emt', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'counselor', type: NounType.Person, confidence: 0.85, isCanonical: false },
// ========== Person - Engineering & Tech ==========
// Canonical
{ keyword: 'engineer', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'developer', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'programmer', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'architect', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'designer', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'technician', type: NounType.Person, confidence: 0.90, isCanonical: true },
// Synonyms
{ keyword: 'coder', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'software engineer', type: NounType.Person, confidence: 0.95, isCanonical: false },
{ keyword: 'software developer', type: NounType.Person, confidence: 0.95, isCanonical: false },
{ keyword: 'web developer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'frontend developer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'backend developer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'full stack developer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'devops engineer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'data engineer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'ml engineer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'machine learning engineer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'data scientist', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'ux designer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'ui designer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'graphic designer', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'systems architect', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'solutions architect', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'tech lead', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'techie', type: NounType.Person, confidence: 0.80, isCanonical: false },
// ========== Person - Management & Leadership ==========
// Canonical
{ keyword: 'manager', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'director', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'executive', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'leader', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'ceo', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'cto', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'cfo', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'coo', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'president', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'founder', type: NounType.Person, confidence: 0.95, isCanonical: true },
// Synonyms
{ keyword: 'supervisor', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'coordinator', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'vp', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'vice president', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'owner', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'product manager', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'project manager', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'engineering manager', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'team lead', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'chief executive officer', type: NounType.Person, confidence: 0.95, isCanonical: false },
{ keyword: 'chief technology officer', type: NounType.Person, confidence: 0.95, isCanonical: false },
{ keyword: 'chief financial officer', type: NounType.Person, confidence: 0.95, isCanonical: false },
// ========== Person - Professional Services ==========
// Canonical
{ keyword: 'analyst', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'consultant', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'specialist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'expert', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'professional', type: NounType.Person, confidence: 0.85, isCanonical: true },
{ keyword: 'lawyer', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'attorney', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'accountant', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'auditor', type: NounType.Person, confidence: 0.90, isCanonical: true },
// Synonyms
{ keyword: 'advisor', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'counselor', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'paralegal', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'legal counsel', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'business analyst', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'financial analyst', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'data analyst', type: NounType.Person, confidence: 0.90, isCanonical: false },
// ========== Person - Education & Research ==========
// Canonical
{ keyword: 'teacher', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'professor', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'researcher', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'scientist', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'student', type: NounType.Person, confidence: 0.95, isCanonical: true },
// Synonyms
{ keyword: 'instructor', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'educator', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'tutor', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'scholar', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'academic', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'pupil', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'learner', type: NounType.Person, confidence: 0.80, isCanonical: false },
{ keyword: 'trainee', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'intern', type: NounType.Person, confidence: 0.85, isCanonical: false },
// ========== Person - Creative Professions ==========
// Canonical
{ keyword: 'artist', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'musician', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'writer', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'author', type: NounType.Person, confidence: 0.90, isCanonical: true },
// Synonyms
{ keyword: 'painter', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'sculptor', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'performer', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'journalist', type: NounType.Person, confidence: 0.90, isCanonical: false },
{ keyword: 'editor', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'reporter', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'content creator', type: NounType.Person, confidence: 0.80, isCanonical: false },
{ keyword: 'blogger', type: NounType.Person, confidence: 0.80, isCanonical: false },
// ========== Person - General ==========
{ keyword: 'person', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'people', type: NounType.Person, confidence: 0.95, isCanonical: true },
{ keyword: 'individual', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'human', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'employee', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'worker', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'staff', type: NounType.Person, confidence: 0.90, isCanonical: true },
{ keyword: 'personnel', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'member', type: NounType.Person, confidence: 0.85, isCanonical: false },
{ keyword: 'team', type: NounType.Person, confidence: 0.80, isCanonical: false },
// Continuing with the rest... (this is getting long, so I'll create a comprehensive version)
// Let me structure this better by importing from the existing typeInference and expanding it
]
// Note: This file will be completed with all 1500+ keywords in the actual implementation
// For now, this shows the structure and approach

50
src/db/db.ts
View file

@ -52,7 +52,7 @@ import type {
Entity,
FindParams,
GetOptions,
GetRelationsParams,
RelatedParams,
Relation,
Result
} from '../types/brainy.types.js'
@ -61,6 +61,7 @@ import {
splitVerbMetadataRecord
} from '../types/reservedFields.js'
import { v4 as uuidv4 } from '../universal/uuid.js'
import { EntityNotFoundError } from '../errors/notFound.js'
import { SpeculativeOverlayError } from './errors.js'
import type { GenerationStore } from './generationStore.js'
import type { ChangedIds, TransactReceipt, TxOperation } from './types.js'
@ -76,8 +77,8 @@ import { entityMatchesFind, resolveEntityField, UnsupportedWhereOperatorError }
export interface HistoricalQueryHandle<T = any> {
/** Full `find()` surface (vector/semantic/graph/cursor/aggregate) at the pinned generation. */
find(query: string | FindParams<T>): Promise<Result<T>[]>
/** Full `getRelations()` surface (including cursor pagination) at the pinned generation. */
getRelations(paramsOrId?: string | GetRelationsParams): Promise<Relation<T>[]>
/** Full `related()` surface (including cursor pagination) at the pinned generation. */
related(paramsOrId?: string | RelatedParams): Promise<Relation<T>[]>
/** Free the materialized indexes (closes the ephemeral reader). Idempotent. */
close(): Promise<void>
}
@ -108,8 +109,8 @@ export interface DbHost<T = any> {
get(id: string, options?: GetOptions): Promise<Entity<T> | null>
/** Live `brain.find()` (current-generation fast path). */
find(query: string | FindParams<T>): Promise<Result<T>[]>
/** Live `brain.getRelations()` (current-generation fast path). */
getRelations(paramsOrId?: string | GetRelationsParams): Promise<Relation<T>[]>
/** Live `brain.related()` (current-generation fast path). */
related(paramsOrId?: string | RelatedParams): Promise<Relation<T>[]>
/** Materialize an entity from a generation record's raw stored objects. */
entityFromRecord(
id: string,
@ -410,7 +411,7 @@ export class Db<T = any> {
/**
* @description Relationships **as of this view's generation** — the `Db`
* counterpart of `brain.getRelations()` (same string-id shorthand and
* counterpart of `brain.related()` (same string-id shorthand and
* filter surface). Relations whose edges changed after the pin are
* resolved from generation records; overlay relations (speculative
* `relate`/`unrelate`) and cascade tombstones (relations touching a
@ -420,18 +421,18 @@ export class Db<T = any> {
* speculative overlay it throws {@link SpeculativeOverlayError}.
*
* @param paramsOrId - An entity id (shorthand for `{ from: id }`) or a
* `GetRelationsParams` filter object.
* `RelatedParams` filter object.
* @returns Relations as of this generation.
*/
async related(paramsOrId?: string | GetRelationsParams): Promise<Relation<T>[]> {
async related(paramsOrId?: string | RelatedParams): Promise<Relation<T>[]> {
this.assertUsable('related')
const params: GetRelationsParams =
const params: RelatedParams =
typeof paramsOrId === 'string' ? { from: paramsOrId } : (paramsOrId ?? {})
const historical = this.isHistorical()
if (!historical && !this.overlay) {
return this.host.getRelations(paramsOrId)
return this.host.related(paramsOrId)
}
if (params.cursor !== undefined) {
@ -439,7 +440,7 @@ export class Db<T = any> {
throw new SpeculativeOverlayError('cursor pagination on related()', this.gen)
}
const materialized = await this.materialize()
return materialized.getRelations(params)
return materialized.related(params)
}
const limit = params.limit ?? 100
@ -451,7 +452,7 @@ export class Db<T = any> {
const overlayVerbs = this.overlay?.verbs ?? new Map<string, Relation<T> | null>()
const excluded = new Set<string>([...changedVerbs, ...overlayVerbs.keys()])
const base = await this.host.getRelations({
const base = await this.host.related({
...params,
limit: limit + offset + excluded.size,
offset: 0
@ -480,7 +481,7 @@ export class Db<T = any> {
// Cascade semantics for speculative deletes: a relation whose endpoint
// is tombstoned in the overlay is gone in this view, exactly as
// brain.delete() cascades on the durable path.
// brain.remove() cascades on the durable path.
if (this.overlay) {
const tombstoned = new Set<string>()
for (const [id, entity] of this.overlay.nouns) {
@ -524,8 +525,8 @@ export class Db<T = any> {
*
* @param ops - The same declarative operations `brain.transact()` accepts.
* @returns A new speculative `Db` over this view.
* @throws Error when an operation references a missing entity/relation,
* mirroring the durable path's planning errors.
* @throws EntityNotFoundError when an operation references a missing
* entity, mirroring the durable path's planning errors.
*/
async with(ops: TxOperation<T>[]): Promise<Db<T>> {
this.assertUsable('with')
@ -587,7 +588,10 @@ export class Db<T = any> {
case 'update': {
const base = await speculativeGet(op.id)
if (!base) {
throw new Error(`with(): entity ${op.id} not found at generation ${this.gen}`)
throw new EntityNotFoundError(
op.id,
`with(): entity ${op.id} not found at generation ${this.gen}`
)
}
// Same reserved-field normalization as the committed update path.
const { reserved, custom } = splitNounMetadataRecord(
@ -631,8 +635,12 @@ export class Db<T = any> {
case 'relate': {
const from = await speculativeGet(op.from)
const to = await speculativeGet(op.to)
if (!from) throw new Error(`with(): source entity ${op.from} not found`)
if (!to) throw new Error(`with(): target entity ${op.to} not found`)
if (!from) {
throw new EntityNotFoundError(op.from, `with(): source entity ${op.from} not found`)
}
if (!to) {
throw new EntityNotFoundError(op.to, `with(): target entity ${op.to} not found`)
}
// Dedupe against the view (overlay first, then the edges visible
// at this generation via the record layer) — mirror of relate().
@ -934,11 +942,11 @@ function sortResultsBy<T>(results: Result<T>[], orderBy: string, order: 'asc' |
}
/**
* @description Filter one relation against `GetRelationsParams`, mirroring
* the storage-level filter `brain.getRelations()` builds (`from` → sourceId,
* @description Filter one relation against `RelatedParams`, mirroring
* the storage-level filter `brain.related()` builds (`from` → sourceId,
* `to` → targetId, type/subtype set membership, service equality).
*/
function relationMatchesParams<T>(relation: Relation<T>, params: GetRelationsParams): boolean {
function relationMatchesParams<T>(relation: Relation<T>, params: RelatedParams): boolean {
if (params.from !== undefined && relation.from !== params.from) return false
if (params.to !== undefined && relation.to !== params.to) return false
if (params.type !== undefined) {

4
src/db/types.ts
View file

@ -53,14 +53,14 @@ export interface TxUpdateOperation<T = any> extends UpdateParams<T> {
}
/**
* @description Delete an entity (and, exactly like `brain.delete()`, every
* @description Remove an entity (and, exactly like `brain.remove()`, every
* relationship where it is source or target — the cascade is part of the
* same atomic batch).
*/
export interface TxRemoveOperation {
/** Discriminator. */
op: 'remove'
/** Id of the entity to delete. */
/** Id of the entity to remove. */
id: string
}

17
src/distributed/coordinator.ts
View file

@ -123,12 +123,13 @@ export class DistributedCoordinator extends EventEmitter {
*/
private setupNetworkHandlers(): void {
if (!this.networkTransport) return
// Register message handlers directly on the messageHandlers map
const handlers = (this.networkTransport as any).messageHandlers as Map<string, (msg: NetworkMessage) => Promise<any>>
// Register handlers through the transport's public onMessage(), which
// writes to the same messageHandlers map this code used to reach into.
const transport = this.networkTransport
// Handle vote requests
handlers.set('requestVote', async (msg: NetworkMessage) => {
transport.onMessage('requestVote', async (msg: NetworkMessage) => {
const response = await this.handleVoteRequest(msg)
// Send response back
if (this.networkTransport) {
@ -138,12 +139,12 @@ export class DistributedCoordinator extends EventEmitter {
})
// Handle vote responses
handlers.set('voteResponse', async (msg: NetworkMessage) => {
transport.onMessage('voteResponse', async (msg: NetworkMessage) => {
this.handleVoteResponse(msg)
})
// Handle heartbeats/append entries
handlers.set('appendEntries', async (msg: NetworkMessage) => {
transport.onMessage('appendEntries', async (msg: NetworkMessage) => {
const response = await this.handleAppendEntries(msg)
// Send response back
if (this.networkTransport) {
@ -153,7 +154,7 @@ export class DistributedCoordinator extends EventEmitter {
})
// Handle append responses
handlers.set('appendResponse', async (msg: NetworkMessage) => {
transport.onMessage('appendResponse', async (msg: NetworkMessage) => {
this.handleAppendResponse(msg)
})
}

7
src/distributed/networkTransport.ts
View file

@ -75,7 +75,12 @@ export class NetworkTransport extends EventEmitter {
if (this.isRunning) return
const ws = await import('ws')
WebSocketServer = (ws as any).WebSocketServer || (ws as any).Server || ws.default
// The ws package has exported its server class as `WebSocketServer`
// (v8+), `Server` (older releases) and via the default export — and the
// available type declarations disagree on which of those are value
// exports, so probe all three through an optional-keyed module view.
const wsModule = ws as typeof ws & { WebSocketServer?: unknown; Server?: unknown }
WebSocketServer = wsModule.WebSocketServer || wsModule.Server || ws.default
await this.startHTTPServer()
await this.startWebSocketServer()

8
src/distributed/storageDiscovery.ts
View file

@ -6,7 +6,7 @@
import { EventEmitter } from 'node:events'
import * as os from 'node:os'
import { StorageAdapter } from '../coreTypes.js'
import { NounMetadata, StorageAdapter } from '../coreTypes.js'
export interface NodeInfo {
id: string
@ -357,7 +357,11 @@ export class StorageDiscovery extends EventEmitter {
*/
private async loadNodeRegistry(): Promise<string[]> {
try {
const registry = await this.storage.getMetadata(`${this.CLUSTER_PATH}/registry.json`) as any
// The registry document is written by updateNodeRegistry() below with a
// `nodes` string-ID array on top of the base NounMetadata shape.
const registry = await this.storage.getMetadata(
`${this.CLUSTER_PATH}/registry.json`
) as (NounMetadata & { nodes?: string[] }) | null
return registry?.nodes || []
} catch (err) {
return []

17
src/embeddings/EmbeddingManager.ts
View file

@ -15,6 +15,17 @@
import { Vector, EmbeddingFunction } from '../coreTypes.js'
import { WASMEmbeddingEngine } from './wasm/index.js'
declare global {
/**
* Unit-test escape hatch set by the test harness (tests/setup-unit.ts):
* when truthy, EmbeddingManager serves deterministic mock embeddings
* instead of loading the real model. Guarded against production use in
* init()/embed()/embedBatch(). Ambient `var` is required here — `let`/
* `const` in `declare global` do not attach to `globalThis`.
*/
var __BRAINY_UNIT_TEST__: boolean | undefined
}
// Types
export type ModelPrecision = 'q8' | 'fp32'
@ -68,7 +79,7 @@ export class EmbeddingManager {
// In unit test mode, skip real model initialization
const isTestMode =
process.env.BRAINY_UNIT_TEST === 'true' ||
(globalThis as any).__BRAINY_UNIT_TEST__
globalThis.__BRAINY_UNIT_TEST__
if (isTestMode) {
// Production safeguard
@ -142,7 +153,7 @@ export class EmbeddingManager {
// Check for unit test environment
const isTestMode =
process.env.BRAINY_UNIT_TEST === 'true' ||
(globalThis as any).__BRAINY_UNIT_TEST__
globalThis.__BRAINY_UNIT_TEST__
if (isTestMode) {
if (process.env.NODE_ENV === 'production') {
@ -224,7 +235,7 @@ export class EmbeddingManager {
const isTestMode =
process.env.BRAINY_UNIT_TEST === 'true' ||
(globalThis as any).__BRAINY_UNIT_TEST__
globalThis.__BRAINY_UNIT_TEST__
if (isTestMode) {
if (process.env.NODE_ENV === 'production') {

79
src/errors/notFound.ts Normal file
View file

@ -0,0 +1,79 @@
/**
* @module errors/notFound
* @description Named not-found errors for Brainy's public contract.
*
* Operations that require an existing record — `update()`, `relate()`,
* `updateRelation()`, `similar({ to: id })`, `transact()` planning, and
* speculative `db.with()` planning — throw these instead of a generic
* `Error`, so callers can branch with `instanceof` and read the missing
* `id` programmatically instead of parsing message strings:
*
* - {@link EntityNotFoundError} — a referenced entity (noun) does not exist
* in the store (or, for `db.with()`, is not visible at the pinned
* generation).
* - {@link RelationNotFoundError} — a referenced relationship (verb) does
* not exist.
*
* Both are exported from the package root (`@soulcraft/brainy`).
*/
/**
* @description Thrown when an operation references an entity id that does
* not exist. Carries the missing {@link EntityNotFoundError.id} so callers
* can recover (re-create, skip, or surface it) without string matching.
*
* @example
* try {
* await brain.update({ id, metadata: { status: 'active' } })
* } catch (err) {
* if (err instanceof EntityNotFoundError) {
* console.log(`missing entity: ${err.id}`)
* }
* }
*/
export class EntityNotFoundError extends Error {
/** The entity id that could not be found. */
public readonly id: string
/**
* @param id - The entity id that could not be found.
* @param message - Optional message override for call sites that add
* context (e.g. source/target role, pinned generation). Defaults to
* `Entity <id> not found`.
*/
constructor(id: string, message?: string) {
super(message ?? `Entity ${id} not found`)
this.name = 'EntityNotFoundError'
this.id = id
}
}
/**
* @description Thrown when an operation references a relationship id that
* does not exist. Carries the missing {@link RelationNotFoundError.id} so
* callers can recover without string matching.
*
* @example
* try {
* await brain.updateRelation({ id: relId, subtype: 'dotted-line' })
* } catch (err) {
* if (err instanceof RelationNotFoundError) {
* console.log(`missing relation: ${err.id}`)
* }
* }
*/
export class RelationNotFoundError extends Error {
/** The relationship id that could not be found. */
public readonly id: string
/**
* @param id - The relationship id that could not be found.
* @param message - Optional message override for call sites that add
* context. Defaults to `Relation <id> not found`.
*/
constructor(id: string, message?: string) {
super(message ?? `Relation ${id} not found`)
this.name = 'RelationNotFoundError'
this.id = id
}
}

6
src/graph/graphAdjacencyIndex.ts
View file

@ -570,9 +570,9 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
for (const [verbId, verb] of loadedVerbs.entries()) {
const cacheKey = `graph:verb:${verbId}`
// Cache the loaded verb with metadata
// Note: HNSWVerbWithMetadata is compatible with GraphVerb (both interfaces)
this.unifiedCache.set(cacheKey, verb as any, 'other', 128, 50) // 128 bytes estimated size, 50ms rebuild cost
results.set(verbId, verb as any)
// Note: HNSWVerbWithMetadata is structurally assignable to GraphVerb
this.unifiedCache.set(cacheKey, verb, 'other', 128, 50) // 128 bytes estimated size, 50ms rebuild cost
results.set(verbId, verb)
}
}

32
src/graph/lsm/LSMTree.ts
View file

@ -145,6 +145,26 @@ class MemTable {
/**
* Manifest - Tracks all SSTables and their levels
*/
/**
* Persisted manifest payload as written by `saveManifest()` (the `data` field
* of the manifest metadata record, after a JSON round-trip).
*/
type PersistedManifestData = {
sstables?: Record<string, number>
lastCompaction?: number
totalRelationships?: number
}
/**
* Persisted SSTable payload as written by `flushMemTable()`/`compact()` (the
* `data` field of an SSTable metadata record): serialized bytes stored as a
* plain number array so they survive JSON round-trips.
*/
type PersistedSSTableData = {
type: string
data: number[]
}
interface Manifest {
/**
* Map of SSTable ID to level
@ -512,10 +532,12 @@ export class LSMTree {
const metadata = await this.storage.getMetadata(`${this.config.storagePrefix}-manifest`)
if (metadata && metadata.data) {
const data = metadata.data as any
// Storage boundary: `data` is the JSON manifest payload written by
// saveManifest(); re-typed from the metadata channel's `unknown`.
const data = metadata.data as PersistedManifestData
this.manifest.sstables = new Map(Object.entries(data.sstables || {}))
this.manifest.lastCompaction = (data.lastCompaction as number) || Date.now()
this.manifest.totalRelationships = (data.totalRelationships as number) || 0
this.manifest.lastCompaction = data.lastCompaction || Date.now()
this.manifest.totalRelationships = data.totalRelationships || 0
// Load SSTables from storage
await this.loadSSTables()
@ -538,7 +560,9 @@ export class LSMTree {
const metadata = await this.storage.getMetadata(storageKey)
if (metadata && metadata.data) {
const data = metadata.data as any
// Storage boundary: `data` is the JSON SSTable payload written by
// flushMemTable()/compact(); re-typed from `unknown`.
const data = metadata.data as PersistedSSTableData
if (data.type === 'lsm-sstable') {
// Convert number[] back to Uint8Array
const uint8Data = new Uint8Array(data.data)

14
src/hnsw/hnswIndex.ts
View file

@ -1228,7 +1228,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
this.clear()
// Step 2: Load system data (entry point, max level)
const systemData = await (this.storage as any).getHNSWSystem()
const systemData = await this.storage.getHNSWSystem()
if (systemData) {
this.entryPointId = systemData.entryPointId
this.maxLevel = systemData.maxLevel
@ -1270,13 +1270,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
{
prodLog.info(`HNSW: Load all nodes at once (${storageType})`)
const result: {
items: HNSWNoun[]
totalCount?: number
hasMore: boolean
nextCursor?: string
} = await (this.storage as any).getNounsWithPagination({
limit: 10000000 // Effectively unlimited for local development
const result = await this.storage.getNounsWithPagination({
limit: 10000000, // Effectively unlimited for local development
offset: 0
})
totalCount = result.totalCount || result.items.length
@ -1285,7 +1281,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
for (const nounData of result.items) {
try {
// Load HNSW graph data for this entity
const hnswData = await (this.storage as any).getVectorIndexData(nounData.id)
const hnswData = await this.storage.getVectorIndexData(nounData.id)
if (!hnswData) {
// No HNSW data - skip (might be entity added before persistence)

13
src/import/BackgroundDeduplicator.ts
View file

@ -123,7 +123,7 @@ export class BackgroundDeduplicator {
try {
// Get all entities from this import using brain.find()
const results = await this.brain.find({
where: { importId } as any,
where: { importId },
limit: 100000 // Large limit to get all entities from import
})
@ -298,7 +298,7 @@ export class BackgroundDeduplicator {
return this.brain.find({
query,
limit: 5,
where: { type: entity.type } as any // Type-aware search
where: { type: entity.type } // Type-aware search
})
})
@ -318,8 +318,11 @@ export class BackgroundDeduplicator {
// Check if not already merged
const stillExists = await this.brain.get(entity.id)
if (stillExists) {
// Cast match.entity to HNSWNounWithMetadata (it comes from brain.find results)
const matchEntity = match.entity as any as HNSWNounWithMetadata
// Typed boundary: bridge the public Entity<T> result shape from
// brain.find() to the storage-layer HNSWNounWithMetadata record
// (same structural bridge as the results.map above). The merge
// path only reads id/type/metadata, present in both shapes.
const matchEntity = match.entity as unknown as HNSWNounWithMetadata
await this.mergeEntities([entity, matchEntity], 'Similarity')
merged++
break // Only merge with first high-similarity match
@ -388,7 +391,7 @@ export class BackgroundDeduplicator {
for (const entity of entities) {
if (entity.id !== primary.id) {
try {
await this.brain.delete(entity.id)
await this.brain.remove(entity.id)
} catch (error) {
// Entity might already be deleted, continue
prodLog.debug(`[BackgroundDedup] Could not delete ${entity.id}:`, error)

6
src/import/EntityDeduplicator.ts
View file

@ -88,7 +88,7 @@ export class EntityDeduplicator {
const results = await this.brain.find({
query: searchText,
limit: 5,
where: opts.strictTypeMatching ? { type: candidate.type } as any : undefined
where: opts.strictTypeMatching ? { type: candidate.type } : undefined
})
// Check each result for potential duplicates
@ -227,7 +227,9 @@ export class EntityDeduplicator {
const entityId = await this.brain.add({
data: candidate.description || candidate.name,
type: candidate.type,
subtype: (candidate as any).subtype ?? 'imported',
subtype:
(candidate as EntityCandidate & { subtype?: string }).subtype ??
'imported',
confidence: candidate.confidence,
metadata: {
...candidate.metadata,

55
src/import/ImportCoordinator.ts
View file

@ -12,7 +12,7 @@
import { Brainy } from '../brainy.js'
import { FormatDetector, SupportedFormat } from './FormatDetector.js'
import { ImportHistory } from './ImportHistory.js'
import { ImportHistory, type ImportHistoryEntry } from './ImportHistory.js'
import { BackgroundDeduplicator } from './BackgroundDeduplicator.js'
import { SmartExcelImporter } from '../importers/SmartExcelImporter.js'
import { SmartPDFImporter } from '../importers/SmartPDFImporter.js'
@ -489,7 +489,13 @@ export class ImportCoordinator {
await this.history.recordImport(
importId,
{
type: normalizedSource.type === 'path' ? 'file' : normalizedSource.type as any,
// 'url' sources were already converted to 'buffer' by
// normalizeSource() → fetchUrl(), so only history-recordable
// source types remain here after the 'path' → 'file' mapping.
type:
normalizedSource.type === 'path'
? 'file'
: (normalizedSource.type as ImportHistoryEntry['source']['type']),
filename: normalizedSource.filename,
format: detection.format
},
@ -699,11 +705,25 @@ export class ImportCoordinator {
format: SupportedFormat,
options: ImportOptions
): Promise<any> {
// Check if IntelligentImportAugmentation already extracted data
if ((options as any)._intelligentImport && (options as any)._extractedData) {
const extractedData = (options as any)._extractedData
// Check if IntelligentImportAugmentation already extracted data.
// Typed boundary: the intelligent-import pre-processing path smuggles its
// results onto ImportOptions via these underscore-prefixed private fields —
// they are not part of the public ImportOptions contract.
const intelligentOptions = options as ImportOptions & {
_intelligentImport?: boolean
_extractedData?: Array<{
id?: string
name?: string
type?: string
description?: string
metadata?: Record<string, unknown>
}>
_metadata?: { intelligentImport?: Record<string, unknown> }
}
if (intelligentOptions._intelligentImport && intelligentOptions._extractedData) {
const extractedData = intelligentOptions._extractedData
// Convert extracted data to ExtractedRow format
const rows = extractedData.map((item: any) => ({
const rows = extractedData.map((item) => ({
entity: {
id: item.id || `entity-${Date.now()}-${Math.random()}`,
name: item.name || item.type || 'Unnamed',
@ -719,7 +739,7 @@ export class ImportCoordinator {
rows,
entities: extractedData,
relationships: [],
metadata: (options as any)._metadata?.intelligentImport || {},
metadata: intelligentOptions._metadata?.intelligentImport || {},
stats: {
byType: {},
byConfidence: {}
@ -811,7 +831,7 @@ export class ImportCoordinator {
entity: {
id: imageId,
name: imageName,
type: 'media' as any,
type: 'media',
description: '',
confidence: 1.0,
metadata: { subtype: 'image' }
@ -864,7 +884,18 @@ export class ImportCoordinator {
provenanceCount?: number
}> {
const entities: Array<{ id: string; name: string; type: NounType; vfsPath?: string; metadata?: Record<string, any> }> = []
const relationships: Array<{ id: string; from: string; to: string; type: VerbType }> = []
// Wider than the declared return type: confidence/weight/metadata are
// collected per relationship for the relateMany() batch below; callers of
// createGraphEntities() only see the narrower {id, from, to, type} rows.
const relationships: Array<{
id: string
from: string
to: string
type: VerbType
confidence?: number
weight?: number
metadata?: { evidence?: string; [key: string]: unknown }
}> = []
let mergedCount = 0
let newCount = 0
@ -1227,7 +1258,7 @@ export class ImportCoordinator {
...trackingContext.customMetadata
})
}
} as any)
})
} catch (error) {
// Skip relationship collection errors (entity might not exist, etc.)
continue
@ -1314,7 +1345,7 @@ export class ImportCoordinator {
verbType = this.inferRelationshipType(
sourceEntity.type,
targetEntity.type,
(rel as any).metadata?.evidence
rel.metadata?.evidence
)
}
@ -1323,7 +1354,7 @@ export class ImportCoordinator {
to: rel.to,
type: verbType, // Enhanced type
metadata: {
...((rel as any).metadata || {}),
...(rel.metadata || {}),
relationshipType: 'semantic', // Distinguish from VFS/provenance
inferredType: verbType !== rel.type, // Track if type was enhanced
originalType: rel.type

2
src/import/ImportHistory.ts
View file

@ -165,7 +165,7 @@ export class ImportHistory {
// Delete entities
for (const entityId of entry.entities) {
try {
await this.brain.delete(entityId)
await this.brain.remove(entityId)
result.entitiesDeleted++
} catch (error) {
result.errors.push(`Failed to delete entity ${entityId}: ${error instanceof Error ? error.message : String(error)}`)

6
src/importers/SmartCSVImporter.ts
View file

@ -164,7 +164,9 @@ export class SmartCSVImporter {
relatedColumn: 'related|see also|links|references',
csvDelimiter: undefined as string | undefined,
csvHeaders: true,
onProgress: () => {},
// Annotated with the full callback signature so the merged opts type has
// a single callable shape (the no-op default narrowed the union).
onProgress: (() => {}) as NonNullable<SmartCSVOptions['onProgress']>,
...options
}
@ -379,7 +381,7 @@ export class SmartCSVImporter {
throughput: Math.round(rowsPerSecond * 10) / 10,
eta: Math.round(estimatedTimeRemaining),
phase: 'extracting'
} as any)
})
}
return {

18
src/importers/SmartExcelImporter.ts
View file

@ -161,15 +161,9 @@ export class SmartExcelImporter {
const opts = {
enableNeuralExtraction: true,
enableRelationshipInference: true,
// CONCEPT EXTRACTION PRODUCTION-READY:
// Type embeddings are now pre-computed at build time - zero runtime cost!
// All 42 noun types + 127 verb types instantly available
//
// Performance profile:
// - Type embeddings: INSTANT (pre-computed at build time, ~100KB in-memory)
// - Model loading: ~2-5 seconds (one-time, cached after first use)
// - Per-row extraction: ~50-200ms depending on definition length
// - 100 rows: ~5-20 seconds total (production ready)
// Type embeddings are pre-computed at build time (~100KB in-memory),
// so concept extraction costs no runtime embedding calls: model loading
// is a one-time ~2-5s, then ~50-200ms per row.
// - 1000 rows: ~50-200 seconds (disable if needed via enableConceptExtraction: false)
//
// Enabled by default for production use.
@ -179,7 +173,9 @@ export class SmartExcelImporter {
definitionColumn: 'definition|description|desc|details',
typeColumn: 'type|category|kind',
relatedColumn: 'related|see also|links',
onProgress: () => {},
// Annotated with the full callback signature so the merged opts type has
// a single callable shape (the no-op default narrowed the union).
onProgress: (() => {}) as NonNullable<SmartExcelOptions['onProgress']>,
...options
}
@ -489,7 +485,7 @@ export class SmartExcelImporter {
throughput: Math.round(rowsPerSecond * 10) / 10,
eta: Math.round(estimatedTimeRemaining),
phase: 'extracting'
} as any)
})
}
// Group rows by sheet for VFS extraction

40
src/importers/SmartImportOrchestrator.ts
View file

@ -133,7 +133,10 @@ export class SmartImportOrchestrator {
const startTime = Date.now()
const result: SmartImportResult = {
success: false,
extraction: null as any,
// Typed boundary: populated in the extraction phase below. If extraction
// throws, the error path returns with this still null (pre-existing
// contract — consumers check `success`/`errors` before reading it).
extraction: null as unknown as SmartExcelResult,
entityIds: [],
relationshipIds: [],
stats: {
@ -193,7 +196,9 @@ export class SmartImportOrchestrator {
const entityId = await this.brain.add({
data: extracted.entity.description,
type: extracted.entity.type,
subtype: (extracted.entity as any).subtype ?? options.defaultSubtype ?? 'imported',
subtype:
(extracted.entity as typeof extracted.entity & { subtype?: string })
.subtype ?? options.defaultSubtype ?? 'imported',
confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata
metadata: {
...extracted.entity.metadata,
@ -280,7 +285,9 @@ export class SmartImportOrchestrator {
from: extracted.entity.id,
to: toEntityId,
type: rel.type,
subtype: (rel as any).subtype ?? options.defaultSubtype ?? 'imported',
subtype:
(rel as typeof rel & { subtype?: string }).subtype ??
options.defaultSubtype ?? 'imported',
metadata: {
confidence: rel.confidence,
evidence: rel.evidence
@ -393,7 +400,8 @@ export class SmartImportOrchestrator {
const startTime = Date.now()
const result: SmartImportResult = {
success: false,
extraction: null as any,
// Typed boundary: populated after extraction (see importExcel).
extraction: null as unknown as SmartExcelResult,
entityIds: [],
relationshipIds: [],
stats: {
@ -413,7 +421,7 @@ export class SmartImportOrchestrator {
const pdfResult = await this.pdfImporter.extract(buffer, options)
// Convert PDF result to Excel-like format for processing
result.extraction = this.convertPDFToExcelFormat(pdfResult) as any
result.extraction = this.convertPDFToExcelFormat(pdfResult)
result.stats.rowsProcessed = pdfResult.sectionsProcessed
// Phase 2 & 3: Create entities and relationships
@ -470,7 +478,8 @@ export class SmartImportOrchestrator {
const startTime = Date.now()
const result: SmartImportResult = {
success: false,
extraction: null as any,
// Typed boundary: populated after extraction (see importExcel).
extraction: null as unknown as SmartExcelResult,
entityIds: [],
relationshipIds: [],
stats: {
@ -488,7 +497,7 @@ export class SmartImportOrchestrator {
const jsonResult = await this.jsonImporter.extract(data, options)
result.extraction = this.convertJSONToExcelFormat(jsonResult) as any
result.extraction = this.convertJSONToExcelFormat(jsonResult)
result.stats.rowsProcessed = jsonResult.nodesProcessed
await this.createEntitiesAndRelationships(result, options, onProgress)
@ -532,7 +541,8 @@ export class SmartImportOrchestrator {
const startTime = Date.now()
const result: SmartImportResult = {
success: false,
extraction: null as any,
// Typed boundary: populated after extraction (see importExcel).
extraction: null as unknown as SmartExcelResult,
entityIds: [],
relationshipIds: [],
stats: {
@ -550,7 +560,7 @@ export class SmartImportOrchestrator {
const mdResult = await this.markdownImporter.extract(markdown, options)
result.extraction = this.convertMarkdownToExcelFormat(mdResult) as any
result.extraction = this.convertMarkdownToExcelFormat(mdResult)
result.stats.rowsProcessed = mdResult.sectionsProcessed
await this.createEntitiesAndRelationships(result, options, onProgress)
@ -601,7 +611,9 @@ export class SmartImportOrchestrator {
const entityId = await this.brain.add({
data: extracted.entity.description,
type: extracted.entity.type,
subtype: (extracted.entity as any).subtype ?? options.defaultSubtype ?? 'imported',
subtype:
(extracted.entity as typeof extracted.entity & { subtype?: string })
.subtype ?? options.defaultSubtype ?? 'imported',
confidence: extracted.entity.confidence, // reserved field — dedicated param, not metadata
metadata: { ...extracted.entity.metadata, name: extracted.entity.name, importedFrom: 'smart-import' }
})
@ -637,7 +649,7 @@ export class SmartImportOrchestrator {
}
// Relationship subtype precedence: extractor → caller default → `'imported'` (7.30.1).
// `confidence` is a reserved top-level field — dedicated relate() param, not metadata
relationshipParams.push({ from: extracted.entity.id, to: toEntityId, type: rel.type, subtype: (rel as any).subtype ?? options.defaultSubtype ?? 'imported', confidence: rel.confidence, metadata: { evidence: rel.evidence } })
relationshipParams.push({ from: extracted.entity.id, to: toEntityId, type: rel.type, subtype: (rel as typeof rel & { subtype?: string }).subtype ?? options.defaultSubtype ?? 'imported', confidence: rel.confidence, metadata: { evidence: rel.evidence } })
} catch (error: any) {
result.errors.push(`Failed to prepare relationship: ${error.message}`)
}
@ -688,7 +700,7 @@ export class SmartImportOrchestrator {
rows,
entityMap: pdfResult.entityMap,
processingTime: pdfResult.processingTime,
stats: pdfResult.stats as any
stats: pdfResult.stats
}
}
@ -710,7 +722,7 @@ export class SmartImportOrchestrator {
rows,
entityMap: jsonResult.entityMap,
processingTime: jsonResult.processingTime,
stats: jsonResult.stats as any
stats: jsonResult.stats
}
}
@ -734,7 +746,7 @@ export class SmartImportOrchestrator {
rows,
entityMap: mdResult.entityMap,
processingTime: mdResult.processingTime,
stats: mdResult.stats as any
stats: mdResult.stats
}
}

10
src/importers/SmartPDFImporter.ts
View file

@ -176,7 +176,9 @@ export class SmartPDFImporter {
minParagraphLength: 50,
extractFromTables: true,
groupBy: 'document' as const,
onProgress: () => {},
// Annotated with the full callback signature so the merged opts type has
// a single callable shape (the no-op default narrowed the union).
onProgress: (() => {}) as NonNullable<SmartPDFOptions['onProgress']>,
...options
}
@ -274,7 +276,7 @@ export class SmartPDFImporter {
throughput: Math.round(sectionsPerSecond * 10) / 10,
eta: Math.round(estimatedTimeRemaining),
phase: 'extracting'
} as any)
})
}
const pagesProcessed = new Set(data.map(d => d._page)).size
@ -338,7 +340,7 @@ export class SmartPDFImporter {
* Process a single section
*/
private async processSection(
group: { id: string, type: string, items: Array<Record<string, any>> },
group: { id: string, type: 'page' | 'paragraph' | 'table', items: Array<Record<string, any>> },
options: SmartPDFOptions,
stats: SmartPDFResult['stats'],
entityMap: Map<string, string>
@ -444,7 +446,7 @@ export class SmartPDFImporter {
return {
sectionId: group.id,
sectionType: group.type as any,
sectionType: group.type,
entities,
relationships,
concepts,

7
src/importers/handlers/excelHandler.ts
View file

@ -69,13 +69,14 @@ export class ExcelHandler extends BaseFormatHandler {
const sheet = workbook.Sheets[sheetName]
if (!sheet) continue
// Convert sheet to JSON with headers
const sheetData = XLSX.utils.sheet_to_json(sheet, {
// Convert sheet to JSON with headers. `header: 1` makes xlsx return
// rows as arrays of raw cell values, typed via the generic overload.
const sheetData = XLSX.utils.sheet_to_json<unknown[]>(sheet, {
header: 1, // Get as array of arrays first
defval: null,
blankrows: false,
raw: false // Convert to formatted strings
}) as any[][]
})
if (sheetData.length === 0) continue

28
src/importers/handlers/pdfHandler.ts
View file

@ -70,8 +70,20 @@ export class PDFHandler extends BaseFormatHandler {
const pdfDoc = await loadingTask.promise
// Extract metadata
// Extract metadata. pdfjs-dist types getMetadata().info as the bare
// `Object` type, so the well-known PDF document-information dictionary
// keys (PDF 32000-1:2008 §14.3.3) are surfaced via a structural type at
// this library boundary.
const metadata = await pdfDoc.getMetadata()
const documentInfo = metadata.info as {
Title?: string
Author?: string
Subject?: string
Creator?: string
Producer?: string
CreationDate?: string
ModDate?: string
} | undefined
const numPages = pdfDoc.numPages
// Report document loaded
@ -177,13 +189,13 @@ export class PDFHandler extends BaseFormatHandler {
textLength: totalTextLength,
tableCount: detectedTables,
pdfMetadata: {
title: (metadata.info as any)?.Title || null,
author: (metadata.info as any)?.Author || null,
subject: (metadata.info as any)?.Subject || null,
creator: (metadata.info as any)?.Creator || null,
producer: (metadata.info as any)?.Producer || null,
creationDate: (metadata.info as any)?.CreationDate || null,
modificationDate: (metadata.info as any)?.ModDate || null
title: documentInfo?.Title || null,
author: documentInfo?.Author || null,
subject: documentInfo?.Subject || null,
creator: documentInfo?.Creator || null,
producer: documentInfo?.Producer || null,
creationDate: documentInfo?.CreationDate || null,
modificationDate: documentInfo?.ModDate || null
}
}
),

19
src/index.ts
View file

@ -1,15 +1,16 @@
/**
* Brainy 3.0 - Your AI-Powered Second Brain
* Brainy 8.0 - Your AI-Powered Second Brain
* 🧠⚛️ A multi-dimensional database with vector, graph, and relational storage
*
*
* Core Components:
* - Brainy: The unified database with Triple Intelligence
* - Triple Intelligence: Seamless fusion of vector + graph + field search
* - Db: Immutable, generation-pinned database values (now/transact/asOf/with)
* - Plugins: Extensible plugin system (cortex, storage adapters)
* - Neural API: AI-powered clustering and analysis
*/
// Export main Brainy class - the modern, clean API for Brainy 3.0
// Export main Brainy class
import { Brainy } from './brainy.js'
export { Brainy }
@ -28,6 +29,14 @@ export type {
RelateParams,
UpdateRelationParams,
FindParams,
SimilarParams,
GetOptions,
RelatedParams,
AddManyParams,
UpdateManyParams,
RemoveManyParams,
RelateManyParams,
BatchResult,
SubtypeRegistry,
FillSubtypeRule,
FillSubtypeRules,
@ -152,6 +161,10 @@ export type { Migration, MigrationState, MigrationPreview, MigrationResult, Migr
// Export optimistic-concurrency types (7.31.0)
export { RevisionConflictError } from './transaction/RevisionConflictError.js'
// Named not-found errors — thrown by update/relate/updateRelation/similar/
// transact/with() when a referenced entity or relation does not exist.
export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js'
// ============= 8.0 Db API — generational MVCC =============
// Immutable database values: brain.now() / brain.transact() / brain.asOf() /
// db.with() / db.persist() / Brainy.load(). See src/db/ for the record layer.

17
src/indexes/columnStore/ColumnManifest.ts
View file

@ -59,9 +59,15 @@ export class ColumnManifest {
async load(storage: StorageAdapter): Promise<void> {
try {
const path = this.manifestPath()
const data = await (storage as any).readObjectFromPath(path)
// readObjectFromPath is a BaseStorage primitive (protected on the
// class, not part of the StorageAdapter interface) — the column store
// owns everything under its base path, and the field-name check below
// validates the manifest before it is adopted.
const data = await (storage as unknown as {
readObjectFromPath: (path: string) => Promise<ManifestData | null>
}).readObjectFromPath(path)
if (data && data.field === this.fieldName) {
this.data = data as ManifestData
this.data = data
}
} catch {
// Not found — start fresh (data is already initialized in constructor)
@ -76,7 +82,12 @@ export class ColumnManifest {
async save(storage: StorageAdapter): Promise<void> {
this.data.version++
const path = this.manifestPath()
await (storage as any).writeObjectToPath(path, this.data)
// writeObjectToPath is a BaseStorage primitive (protected on the class,
// not part of the StorageAdapter interface) — same typed boundary as
// load() above.
await (storage as unknown as {
writeObjectToPath: (path: string, data: ManifestData) => Promise<void>
}).writeObjectToPath(path, this.data)
}
/**

15
src/indexes/columnStore/ColumnStore.ts
View file

@ -120,7 +120,12 @@ export class ColumnStore implements ColumnStoreProvider {
// Discover fields by listing manifest files
try {
const paths = await (storage as any).listObjectsUnderPath(this.basePath + '/')
// listObjectsUnderPath is a BaseStorage primitive (protected on the
// class, not part of the StorageAdapter interface) — same typed
// boundary as the blob reads below.
const paths = await (storage as unknown as {
listObjectsUnderPath: (prefix: string) => Promise<string[]>
}).listObjectsUnderPath(this.basePath + '/')
for (const path of paths) {
if (path.endsWith('/MANIFEST.json')) {
const fieldName = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '')
@ -637,8 +642,12 @@ export class ColumnStore implements ColumnStoreProvider {
if (!buffer || buffer.size === 0) return null
const valueType = this.fieldTypes.get(field) ?? ValueType.String
// Drain a COPY for querying — don't clear the buffer
const entries = (buffer as any).entries as Array<{ value: number | string, entityIntId: number }>
// Drain a COPY for querying — don't clear the buffer. This reads the
// buffer's private accumulator directly (drain() is the only public
// accessor and it clears the buffer, which queries must not do).
const entries = (buffer as unknown as {
entries: Array<{ value: number | string, entityIntId: number }>
}).entries
if (!entries || entries.length === 0) return null
// Sort a copy for the cursor

4
src/integrations/core/IntegrationBase.ts
View file

@ -265,7 +265,7 @@ export abstract class IntegrationBase {
}
/**
* Query relations using Brainy getRelations()
* Query relations using Brainy related()
*/
protected async queryRelations(params?: {
from?: string
@ -278,7 +278,7 @@ export abstract class IntegrationBase {
throw new Error('Integration not initialized')
}
return this.context.brain.getRelations(params)
return this.context.brain.related(params)
}
/**

17
src/integrations/core/IntegrationHub.ts
View file

@ -82,7 +82,12 @@ export interface IntegrationResponse {
export class IntegrationHub {
private integrations: Map<IntegrationType, IntegrationBase> = new Map()
private config: Required<IntegrationHubConfig>
private _endpoints: Record<IntegrationType, string> = {} as any
/**
* Endpoint paths keyed by integration type. Starts empty and gains one key
* per integration enabled in initialize() — disabled integrations have no
* entry, hence the sparse-record assertion on the initializer.
*/
private _endpoints: Record<IntegrationType, string> = {} as Record<IntegrationType, string>
/**
* Create and initialize the Integration Hub
@ -168,7 +173,7 @@ export class IntegrationHub {
private getBasePath(type: IntegrationType): string {
const basePath = this.config.basePath
const cfg = this.config.config?.[type] as any
const cfg = this.config.config?.[type]
switch (type) {
case 'odata':
@ -222,7 +227,13 @@ export class IntegrationHub {
if (path.startsWith(basePath) || path === basePath) {
const integration = this.integrations.get(type as IntegrationType)
if (integration && 'handleRequest' in integration) {
const handler = integration as any
// The `in` guard proved the integration exposes an HTTP handler.
// Each integration declares its own structurally-compatible
// request/response shapes (e.g. the OData request/response pair),
// so the hub dispatches through this minimal common signature.
const handler = integration as IntegrationBase & {
handleRequest(request: IntegrationRequest): Promise<IntegrationResponse>
}
const relativePath = path.slice(basePath.length) || '/'
return handler.handleRequest({

17
src/integrations/core/IntegrationLoader.ts
View file

@ -76,20 +76,29 @@ export const INTEGRATION_CATALOG: Record<IntegrationType, IntegrationInfo> = {
* Detect current runtime environment
*/
export function detectEnvironment(): RuntimeEnvironment {
// Runtime-detection boundary: these globals exist only in specific runtimes
// (Deno, Bun, Cloudflare Workers), so they aren't on TypeScript's view of
// globalThis — probe them through an optional-keyed view.
const runtimeGlobals = globalThis as typeof globalThis & {
Deno?: unknown
Bun?: unknown
HTMLRewriter?: unknown
}
// Deno
if (typeof (globalThis as any).Deno !== 'undefined') {
if (typeof runtimeGlobals.Deno !== 'undefined') {
return 'deno'
}
// Bun
if (typeof (globalThis as any).Bun !== 'undefined') {
if (typeof runtimeGlobals.Bun !== 'undefined') {
return 'bun'
}
// Cloudflare Workers
if (
typeof (globalThis as any).caches !== 'undefined' &&
typeof (globalThis as any).HTMLRewriter !== 'undefined'
typeof runtimeGlobals.caches !== 'undefined' &&
typeof runtimeGlobals.HTMLRewriter !== 'undefined'
) {
return 'cloudflare'
}

7
src/integrations/core/TabularExporter.ts
View file

@ -6,6 +6,7 @@
*/
import { Entity, Relation } from '../../types/brainy.types.js'
import { NounType } from '../../types/graphTypes.js'
import {
TabularRow,
RelationTabularRow,
@ -462,9 +463,11 @@ export class TabularExporter {
private rowToEntity(row: Record<string, any>): Partial<Entity> {
const entity: Partial<Entity> = {}
// Map standard columns
// Map standard columns. Imported rows are an untrusted boundary — the
// Type cell is taken at its word here; Brainy validates the NounType
// vocabulary when the partial entity is written.
if (row.Id) entity.id = row.Id
if (row.Type) entity.type = row.Type as any
if (row.Type) entity.type = row.Type as NounType
if (row.Service) entity.service = row.Service
if (row.Confidence) entity.confidence = parseFloat(row.Confidence)
if (row.Weight) entity.weight = parseFloat(row.Weight)

24
src/integrations/odata/ODataIntegration.ts
View file

@ -14,7 +14,12 @@ import {
IntegrationBase,
HTTPIntegration
} from '../core/IntegrationBase.js'
import { IntegrationConfig, ODataQueryOptions } from '../core/types.js'
import {
IntegrationConfig,
ODataQueryOptions,
RelationTabularRow,
TabularRow
} from '../core/types.js'
import { Entity, Relation, FindParams } from '../../types/brainy.types.js'
import { NounType } from '../../types/graphTypes.js'
import {
@ -387,12 +392,13 @@ export class ODataIntegration extends IntegrationBase implements HTTPIntegration
entities = entities.slice(0, pageSize)
}
// Convert to tabular format
let rows = this.exporter.entitiesToRows(entities)
// Convert to tabular format. $select projects rows down to a subset of
// columns, so the working type is Partial<TabularRow> from the start.
let rows: Array<Partial<TabularRow>> = this.exporter.entitiesToRows(entities)
// Apply $select
if (options.select && options.select.length > 0) {
rows = applySelect(rows, options.select) as any
rows = applySelect(rows, options.select)
}
// Apply $orderby (if not handled by storage)
@ -528,7 +534,7 @@ export class ODataIntegration extends IntegrationBase implements HTTPIntegration
return this.errorResponse(400, 'Invalid entity ID')
}
await this.context.brain.delete(idMatch[1])
await this.context.brain.remove(idMatch[1])
return {
status: 204,
@ -566,16 +572,18 @@ export class ODataIntegration extends IntegrationBase implements HTTPIntegration
offset: options.skip
})
let rows = this.exporter.relationsToRows(relations)
// $select projects rows down to a subset of columns, so the working type
// is Partial<RelationTabularRow> from the start.
let rows: Array<Partial<RelationTabularRow>> = this.exporter.relationsToRows(relations)
// Apply $filter
if (options.filter) {
rows = applyFilter(rows, options.filter) as any
rows = applyFilter(rows, options.filter)
}
// Apply $select
if (options.select && options.select.length > 0) {
rows = applySelect(rows, options.select) as any
rows = applySelect(rows, options.select)
}
// Apply $orderby

4
src/integrations/sheets/GoogleSheetsIntegration.ts
View file

@ -552,7 +552,7 @@ export class GoogleSheetsIntegration
return this.errorResponse(400, 'Missing required field: id')
}
await this.context.brain.delete(body.id)
await this.context.brain.remove(body.id)
return this.jsonResponse({ success: true, deleted: body.id })
}
@ -595,7 +595,7 @@ export class GoogleSheetsIntegration
break
case 'delete':
await this.context.brain.delete(op.id)
await this.context.brain.remove(op.id)
results.push({ success: true, id: op.id, action: 'delete' })
break

11
src/integrations/sse/SSEIntegration.ts
View file

@ -338,20 +338,23 @@ export class SSEIntegration
): EventFilter {
const filter: EventFilter = {}
// HTTP query strings are an untrusted boundary: narrow them to the typed
// filter fields directly. Unknown values are harmless — matchesFilter()
// compares with includes(), so they simply never match an event.
if (query.types) {
filter.entityTypes = query.types.split(',') as any[]
filter.entityTypes = query.types.split(',') as EventFilter['entityTypes']
}
if (query.operations) {
filter.operations = query.operations.split(',') as any[]
filter.operations = query.operations.split(',') as EventFilter['operations']
}
if (query.nounTypes) {
filter.nounTypes = query.nounTypes.split(',') as any[]
filter.nounTypes = query.nounTypes.split(',') as EventFilter['nounTypes']
}
if (query.verbTypes) {
filter.verbTypes = query.verbTypes.split(',') as any[]
filter.verbTypes = query.verbTypes.split(',') as EventFilter['verbTypes']
}
if (query.service) {

200
src/interfaces/IIndex.ts
View file

@ -1,200 +0,0 @@
/**
* Unified Index Interface
*
* Standardizes index lifecycle across all index types in Brainy.
* All indexes (HNSW Vector, Graph Adjacency, Metadata Field) implement this interface
* for consistent rebuild, clear, and stats operations.
*
* This enables:
* - Parallel index rebuilds during initialization
* - Consistent index management across the system
* - Easy addition of new index types
* - Unified monitoring and health checks
*/
/**
* Index statistics returned by getStats()
*/
export interface IndexStats {
/**
* Total number of items in the index
*/
totalItems: number
/**
* Estimated memory usage in bytes (optional)
*/
memoryUsage?: number
/**
* Timestamp of last rebuild (optional)
*/
lastRebuilt?: number
/**
* Index-specific statistics (optional)
* - HNSW: { maxLevel, entryPointId, levels, avgDegree }
* - Graph: { totalRelationships, verbTypes }
* - Metadata: { totalFields, totalEntries }
*/
specifics?: Record<string, any>
}
/**
* Progress callback for rebuild operations
* Reports current progress and total count
*/
export type RebuildProgressCallback = (loaded: number, total: number) => void
/**
* Rebuild options for index rebuilding
*/
export interface RebuildOptions {
/**
* Batch size for pagination during rebuild
* Default: 1000 (tune based on available memory)
*/
batchSize?: number
/**
* Progress callback for monitoring rebuild progress
* Called periodically with (loaded, total) counts
*/
onProgress?: RebuildProgressCallback
/**
* Force rebuild even if index appears populated
* Useful for repairing corrupted indexes
*/
force?: boolean
}
/**
* Unified Index Interface
*
* All indexes in Brainy implement this interface for consistent lifecycle management.
* This enables parallel rebuilds, unified monitoring, and standardized operations.
*/
export interface IIndex {
/**
* Rebuild index from persisted storage
*
* Called during Brainy initialization when:
* - Container restarts and in-memory indexes are empty
* - Storage has persisted data but indexes need rebuilding
* - Force rebuild is requested
*
* Implementation must:
* - Clear existing in-memory state
* - Load data from storage using pagination
* - Restore index structure efficiently (O(N) preferred over O(N log N))
* - Handle millions of entities via batching
* - Auto-detect caching strategy based on dataset size vs available memory
* - Provide progress reporting for large datasets
* - Recover gracefully from partial failures
*
* Adaptive Caching:
* System automatically chooses optimal strategy:
* - Small datasets: Preload all data at init for zero-latency access
* - Large datasets: Load on-demand via UnifiedCache for memory efficiency
*
* @param options Rebuild options (batch size, progress callback, force)
* @returns Promise that resolves when rebuild is complete
* @throws Error if rebuild fails critically (should log warnings for partial failures)
*/
rebuild(options?: RebuildOptions): Promise<void>
/**
* Clear all in-memory index data
*
* Called when:
* - User explicitly calls brain.clear()
* - System needs to reset without rebuilding
* - Tests need clean state
*
* Implementation must:
* - Clear all in-memory data structures
* - Reset counters and statistics
* - NOT delete persisted storage data
* - Be idempotent (safe to call multiple times)
*
* Note: This is a memory-only operation. To delete persisted data,
* use storage.clear() instead.
*/
clear(): void
/**
* Get current index statistics
*
* Returns real-time statistics about the index state:
* - Total items indexed
* - Memory usage (if available)
* - Last rebuild timestamp
* - Index-specific metrics
*
* Used for:
* - Health monitoring
* - Determining if rebuild is needed
* - Performance analysis
* - Debugging
*
* @returns Promise that resolves to index statistics
*/
getStats(): Promise<IndexStats>
/**
* Get the current size of the index
*
* Fast O(1) operation returning the number of items in the index.
* Used for quick health checks and deciding rebuild strategy.
*
* @returns Number of items in the index
*/
size(): number
}
/**
* Extended index interface with cache support (optional)
*
* Indexes can optionally implement cache integration for:
* - Hot/warm/cold tier management
* - Memory-efficient lazy loading
* - Adaptive caching based on access patterns
*/
export interface ICachedIndex extends IIndex {
/**
* Set cache for resource management
*
* Enables the index to use UnifiedCache for:
* - Lazy loading of vectors/data
* - Hot/warm/cold tier management
* - Memory pressure handling
*
* @param cache UnifiedCache instance
*/
setCache?(cache: any): void
}
/**
* Extended index interface with persistence support (optional)
*
* Indexes can optionally implement explicit persistence:
* - Manual triggering of data saves
* - Batch write optimization
* - Checkpoint creation
*/
export interface IPersistentIndex extends IIndex {
/**
* Manually persist current index state to storage
*
* Most indexes auto-persist during operations (e.g., HNSW persists on addItem).
* This method allows explicit persistence for:
* - Checkpointing before risky operations
* - Forced flush before shutdown
* - Manual backup creation
*
* @returns Promise that resolves when persistence is complete
*/
persist?(): Promise<void>
}

13
src/mcp/brainyMCPAdapter.ts
View file

@ -146,10 +146,15 @@ export class BrainyMCPAdapter {
)
}
// This is a simplified implementation - in a real implementation, we would
// need to check if these methods exist on the BrainyInterface
const outgoing = await (this.brainyData as any).getVerbsBySource?.(id) || []
const incoming = await (this.brainyData as any).getVerbsByTarget?.(id) || []
// BrainyInterface doesn't expose verb-level accessors — these optional
// probes target methods older Brainy builds carried, and resolve to empty
// relationship lists when (as on current builds) the methods are absent.
const verbAccessors = this.brainyData as BrainyInterface & {
getVerbsBySource?: (id: string) => Promise<unknown[]>
getVerbsByTarget?: (id: string) => Promise<unknown[]>
}
const outgoing = await verbAccessors.getVerbsBySource?.(id) || []
const incoming = await verbAccessors.getVerbsByTarget?.(id) || []
return this.createSuccessResponse(request.requestId, { outgoing, incoming })
}

4
src/neural/SmartExtractor.ts
View file

@ -1,7 +1,7 @@
/**
* SmartExtractor - Unified entity type extraction using ensemble of neural signals
*
* PRODUCTION-READY: Single orchestration class for all entity type classification
* Single orchestration class for all entity type classification
*
* Design Philosophy:
* - Simplicity over complexity (KISS principle)
@ -645,7 +645,7 @@ export class SmartExtractor {
return {
type: best.type!,
confidence: best.confidence,
source: best.signal as any,
source: best.signal,
evidence: best.evidence,
metadata: {
formatHints: formatHints.length > 0 ? formatHints : undefined,

2
src/neural/SmartRelationshipExtractor.ts
View file

@ -1,7 +1,7 @@
/**
* SmartRelationshipExtractor - Unified relationship type extraction using ensemble of neural signals
*
* PRODUCTION-READY: Parallel to SmartExtractor but for verbs/relationships
* Parallel to SmartExtractor but for verbs/relationships
*
* Design Philosophy:
* - Simplicity over complexity (KISS principle)

4
src/neural/embeddedPatterns.ts
View file

File diff suppressed because one or more lines are too long

2
src/neural/entityExtractionCache.ts
View file

@ -3,8 +3,6 @@
*
* Caches entity extraction results to avoid re-processing unchanged content.
* Uses file mtime or content hash for invalidation.
*
* PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
*/
import { ExtractedEntity } from './entityExtractor.js'

12
src/neural/entityExtractor.ts
View file

@ -3,7 +3,7 @@
* Uses embeddings and similarity matching for accurate type detection
*
* Now powered by SmartExtractor for ultra-neural classification
* PRODUCTION-READY with caching support
* Entity extraction with caching support
*/
import { NounType } from '../types/graphTypes.js'
@ -293,8 +293,8 @@ export class NeuralEntityExtractor {
const contextLower = context.toLowerCase()
let boost = 0
// Context clues for each type
const contextClues: Record<NounType, string[]> = {
// Context clues for each type (partial: only types with known clue words)
const contextClues: Partial<Record<NounType, string[]>> = {
[NounType.Person]: ['mr', 'ms', 'mrs', 'dr', 'prof', 'said', 'told', 'wrote'],
[NounType.Organization]: ['inc', 'corp', 'llc', 'ltd', 'company', 'announced'],
[NounType.Location]: ['in', 'at', 'from', 'to', 'near', 'located', 'city', 'country'],
@ -304,7 +304,7 @@ export class NeuralEntityExtractor {
[NounType.Currency]: ['$', '€', '£', '¥', 'usd', 'eur', 'price', 'cost'],
[NounType.Message]: ['email', 'message', 'sent', 'received', 'wrote', 'reply'],
// Add more context clues as needed
} as any
}
const clues = contextClues[type] || []
for (const clue of clues) {
@ -401,8 +401,8 @@ export class NeuralEntityExtractor {
this.embeddingCacheStats.misses++
let vector: Vector
if ('embed' in this.brain && typeof (this.brain as any).embed === 'function') {
vector = await (this.brain as any).embed(text)
if ('embed' in this.brain && typeof this.brain.embed === 'function') {
vector = await this.brain.embed(text)
} else {
// Fallback - create simple hash-based vector
vector = new Array(384).fill(0)

21
src/neural/improvedNeuralAPI.ts
View file

@ -1003,8 +1003,12 @@ export class ImprovedNeuralAPI {
return this._createEmptyResult(startTime, 'hierarchical')
}
// Use existing HNSW level structure for natural clustering
const level = (options as any).level || this._getOptimalClusteringLevel(itemIds.length)
// Use existing HNSW level structure for natural clustering.
// `level` is an undocumented expert override (not on ClusteringOptions):
// it pins the HNSW layer used for cluster centers instead of deriving it.
const level =
(options as ClusteringOptions & { level?: number }).level ||
this._getOptimalClusteringLevel(itemIds.length)
const maxClusters = options.maxClusters || Math.min(50, Math.ceil(itemIds.length / 20))
// Get HNSW level representatives - these are natural cluster centers
@ -1618,7 +1622,7 @@ export class ImprovedNeuralAPI {
// Get all verbs connecting the items
for (const sourceId of itemIds) {
const sourceVerbs = await this.brain.getRelations(sourceId)
const sourceVerbs = await this.brain.related(sourceId)
for (const verb of sourceVerbs) {
const targetId = verb.to
@ -1882,8 +1886,9 @@ export class ImprovedNeuralAPI {
return await this.brain.counts.entities()
}
// Fallback: Get from storage statistics
const storage = (this.brain as any).storage
// Fallback: Get from storage statistics (brain is untyped here; storage is
// a private Brainy field reached for its optional getStatistics capability)
const storage = this.brain.storage
if (storage && typeof storage.getStatistics === 'function') {
const stats = await storage.getStatistics()
return stats?.totalNodes || 0
@ -2836,7 +2841,7 @@ export class ImprovedNeuralAPI {
private async _getImportantSample(itemIds: string[], sampleSize: number): Promise<string[]> {
const items = await Promise.all(
itemIds.map(async id => {
const verbs = await this.brain.getRelations(id)
const verbs = await this.brain.related(id)
const noun = await this.brain.get(id)
// Calculate importance score
@ -3436,7 +3441,7 @@ export class ImprovedNeuralAPI {
const sampleSize = Math.min(50, itemIds.length)
for (let i = 0; i < sampleSize; i++) {
try {
const verbs = await this.brain.getRelations({ from: itemIds[i] })
const verbs = await this.brain.related({ from: itemIds[i] })
connectionCount += verbs.length
} catch (error) {
// Skip items that can't be processed
@ -3506,7 +3511,7 @@ export class ImprovedNeuralAPI {
if (fromType !== toType) {
for (const fromItem of fromItems.slice(0, 10)) { // Sample to avoid N^2
try {
const verbs = await this.brain.getRelations({ from: fromItem.id })
const verbs = await this.brain.related({ from: fromItem.id })
for (const verb of verbs) {
const toItem = toItems.find(item => item.id === verb.to)

10
src/neural/naturalLanguageProcessor.ts
View file

@ -78,7 +78,7 @@ export class NaturalLanguageProcessor {
}
// Use brain's embed method directly to avoid recursion
const embedding = await (this.brain as any).embed(text)
const embedding = await this.brain.embed(text)
// Cache the embedding
this.embeddingCache.set(text, embedding)
@ -1161,8 +1161,8 @@ export class NaturalLanguageProcessor {
const extractor = new NeuralEntityExtractor(this.brain)
// Convert string types to NounTypes if provided
const nounTypes = options?.types ?
options.types.map(t => t as any) :
const nounTypes = options?.types ?
options.types.map(t => t as NounType) :
undefined
// Extract using neural matching
@ -1253,7 +1253,7 @@ export class NaturalLanguageProcessor {
const confidence = this.calculateConfidence(extractedText, type)
if (confidence >= minConfidence) {
const entity = {
const entity: (typeof extracted)[number] = {
text: extractedText,
type,
position: {
@ -1264,7 +1264,7 @@ export class NaturalLanguageProcessor {
}
if (options?.includeMetadata) {
(entity as any).metadata = {
entity.metadata = {
pattern: pattern.source,
contextBefore: text.substring(Math.max(0, match.index - 20), match.index),
contextAfter: text.substring(match.index + match[0].length, Math.min(text.length, match.index + match[0].length + 20))

4
src/neural/neuralAPI.ts
View file

@ -646,7 +646,7 @@ export class NeuralAPI {
// Run k-means iterations (simplified)
for (let iter = 0; iter < 10; iter++) {
const clusters = Array(k).fill(null).map(() => [])
const clusters: (typeof items)[] = Array(k).fill(null).map(() => [])
// Assign items to nearest centroid
for (const item of items) {
@ -661,7 +661,7 @@ export class NeuralAPI {
}
}
(clusters as any[])[bestCluster].push(item)
clusters[bestCluster].push(item)
}
// Update centroids

14
src/neural/neuralImport.ts
View file

@ -36,6 +36,11 @@ export interface DetectedEntity {
suggestedId: string
reasoning: string
alternativeTypes: Array<{ type: string, confidence: number }>
/**
* Subtype tag set by the extractor, if any. Takes precedence over
* `NeuralImportOptions.defaultSubtype` during import execution.
*/
subtype?: string
}
export interface DetectedRelationship {
@ -47,6 +52,11 @@ export interface DetectedRelationship {
reasoning: string
context: string
metadata?: Record<string, any>
/**
* Subtype tag set by the extractor, if any. Takes precedence over
* `NeuralImportOptions.defaultSubtype` during import execution.
*/
subtype?: string
}
export interface NeuralInsight {
@ -791,7 +801,7 @@ export class NeuralImport {
await this.brainy.add({
data: this.extractMainText(entity.originalData),
type: entity.nounType as NounType,
subtype: (entity as any).subtype ?? options.defaultSubtype ?? 'extracted',
subtype: entity.subtype ?? options.defaultSubtype ?? 'extracted',
metadata: {
...entity.originalData,
confidence: entity.confidence,
@ -806,7 +816,7 @@ export class NeuralImport {
from: relationship.sourceId,
to: relationship.targetId,
type: relationship.verbType as VerbType,
subtype: (relationship as any).subtype ?? options.defaultSubtype ?? 'extracted',
subtype: relationship.subtype ?? options.defaultSubtype ?? 'extracted',
weight: relationship.weight,
confidence: relationship.confidence, // reserved field — dedicated param, not metadata
metadata: {

2
src/neural/neuralImportAugmentation.ts
View file

@ -92,7 +92,7 @@ export class NeuralImportAugmentation {
next: () => Promise<T>
): Promise<T> {
// Only process on add operations
if (!this.operations.includes(operation as any)) {
if (!this.operations.includes(operation)) {
return next()
}

2
src/neural/patternLibrary.ts
View file

@ -119,7 +119,7 @@ export class PatternLibrary {
}
// Use brain's embed method directly to avoid recursion
const embedding = await (this.brain as any).embed(text)
const embedding = await this.brain.embed(text)
this.embeddingCache.set(text, embedding)
return embedding

79
src/neural/patterns.ts
View file

@ -1,79 +0,0 @@
/**
* Core Pattern Library with Pre-computed Embeddings
*
* This file is auto-generated by scripts/buildPatterns.ts
* DO NOT EDIT MANUALLY - edit src/patterns/comprehensive-library.json instead
*
* Storage strategy:
* - Patterns are bundled directly into Brainy for zero-latency access
* - Embeddings are pre-computed and stored as binary Float32Array
* - Total size: ~140KB (negligible for a neural library)
* - No external files needed, works in all environments
*/
import type { Pattern } from './patternLibrary.js'
// Pattern data embedded directly for reliability
export const CORE_PATTERNS: Pattern[] = [
// Informational queries
{
id: "info_what_is",
category: "informational",
examples: ["what is artificial intelligence", "what is machine learning"],
pattern: "what is (.+)",
template: { like: "${1}" },
confidence: 0.9
},
{
id: "info_how_does",
category: "informational",
examples: ["how does neural network work", "how does deep learning work"],
pattern: "how does (.+) work",
template: { like: "${1}" },
confidence: 0.85
},
// ... more patterns loaded from library.json at build time
]
// Pre-computed embeddings as binary data
// Generated by scripts/buildPatterns.ts using Brainy's embedding model
export const PATTERN_EMBEDDINGS_BINARY: Uint8Array | null = null // Will be populated at build
// Helper to decode embeddings
export function getPatternEmbeddings(): Map<string, Float32Array> {
if (!PATTERN_EMBEDDINGS_BINARY) {
return new Map() // Will compute at runtime if not pre-built
}
const embeddings = new Map<string, Float32Array>()
const view = new DataView(PATTERN_EMBEDDINGS_BINARY.buffer)
const embeddingSize = 384 // Standard size
CORE_PATTERNS.forEach((pattern, index) => {
const offset = index * embeddingSize * 4 // 4 bytes per float
const embedding = new Float32Array(embeddingSize)
for (let i = 0; i < embeddingSize; i++) {
embedding[i] = view.getFloat32(offset + i * 4, true)
}
embeddings.set(pattern.id, embedding)
})
return embeddings
}
// Version for cache invalidation
export const PATTERNS_VERSION = "2.0.0"
// Export metadata for monitoring
export const PATTERNS_METADATA = {
totalPatterns: CORE_PATTERNS.length,
categories: [...new Set(CORE_PATTERNS.map(p => p.category))],
embeddingDimensions: 384,
storageSize: {
patterns: "24KB",
embeddings: "98KB",
total: "122KB"
}
}

2
src/neural/relationshipConfidence.ts
View file

@ -6,8 +6,6 @@
* - Entity confidence scores
* - Pattern matches
* - Structural analysis
*
* PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
*/
import { ExtractedEntity } from './entityExtractor.js'

2
src/neural/signals/ContextSignal.ts
View file

@ -1,7 +1,7 @@
/**
* ContextSignal - Relationship-based entity type classification
*
* PRODUCTION-READY: Infers types from relationship patterns in context
* Infers types from relationship patterns in context
*
* Weight: 5% (lowest, but critical for ambiguous cases)
* Speed: Very fast (~1ms) - pure pattern matching on context

6
src/neural/signals/EmbeddingSignal.ts
View file

@ -1,7 +1,7 @@
/**
* EmbeddingSignal - Neural entity type classification using embeddings
*
* PRODUCTION-READY: Merges neural + graph + temporal signals into one
* Merges neural + graph + temporal signals into one
* 3x faster than separate signals (single embedding lookup)
*
* Weight: 35% (20% neural + 10% graph + 5% temporal boost)
@ -56,7 +56,7 @@ export interface EmbeddingSignalOptions {
interface SourceMatch {
type: NounType
confidence: number
source: string
source: TypeSignal['source']
metadata?: any
}
@ -390,7 +390,7 @@ export class EmbeddingSignal {
}
return {
source: bestMatches.length > 1 ? 'embedding-combined' : bestMatches[0].source as any,
source: bestMatches.length > 1 ? 'embedding-combined' : bestMatches[0].source,
type: bestType,
confidence: Math.min(bestCombinedScore, 1.0), // Cap at 1.0
evidence,

1
src/neural/signals/PatternSignal.ts
View file

@ -11,7 +11,6 @@
* Merges: KeywordSignal + PatternSignal from old architecture
* Speed: Very fast (~5ms) - pre-compiled patterns
*
* PRODUCTION-READY: No TODOs, no mocks, real implementation
*/
import type { Brainy } from '../../brainy.js'

1
src/neural/signals/VerbContextSignal.ts
View file

@ -8,7 +8,6 @@
* 2. Semantic compatibility (Document+Person → CreatedBy)
* 3. Domain heuristics (Location+Organization → LocatedAt)
*
* PRODUCTION-READY: No TODOs, no mocks, real implementation
*/
import type { Brainy } from '../../brainy.js'

8
src/neural/signals/VerbEmbeddingSignal.ts
View file

@ -8,7 +8,6 @@
* 2. Cosine similarity against context text
* 3. Semantic understanding of relationship intent
*
* PRODUCTION-READY: No TODOs, no mocks, real implementation
*/
import type { Brainy } from '../../brainy.js'
@ -89,7 +88,12 @@ export class VerbEmbeddingSignal {
constructor(brain: Brainy, options?: VerbEmbeddingSignalOptions) {
this.brain = brain
this.distanceFn = (brain as any).distance || cosineDistance
// Visibility boundary: Brainy keeps its distance function in a private
// field. The signal reuses it when present so verb classification matches
// the index geometry, falling back to cosine distance otherwise.
this.distanceFn =
(brain as unknown as { distance?: (a: Vector, b: Vector) => number }).distance ||
cosineDistance
this.options = {
minConfidence: options?.minConfidence ?? 0.60,
minSimilarity: options?.minSimilarity ?? 0.55,

1
src/neural/signals/VerbPatternSignal.ts
View file

@ -8,7 +8,6 @@
* 2. Prepositional phrase patterns ("in", "at", "by", "of")
* 3. Structural patterns (parentheses, commas, formatting)
*
* PRODUCTION-READY: No TODOs, no mocks, real implementation
*/
import type { Brainy } from '../../brainy.js'

8
src/neural/staticPatternMatcher.ts
View file

@ -14,9 +14,11 @@ const patterns = new Map(EMBEDDED_PATTERNS.map(p => [p.id, p]))
const patternEmbeddings = getPatternEmbeddings()
/**
* Cosine similarity between two vectors
* Cosine similarity between two vectors.
* Accepts any number-indexed sequence so pre-computed Float32Array pattern
* embeddings can be compared against number[] query vectors without copying.
*/
function cosineSimilarity(a: Vector, b: Vector): number {
function cosineSimilarity(a: ArrayLike<number>, b: ArrayLike<number>): number {
if (!a || !b || a.length !== b.length) return 0
let dotProduct = 0
@ -99,7 +101,7 @@ export function findBestPatterns(
if (!patternEmbedding) continue
// Pass Float32Array directly, no need for Array.from()!
const similarity = cosineSimilarity(queryEmbedding, patternEmbedding as any)
const similarity = cosineSimilarity(queryEmbedding, patternEmbedding)
if (similarity > 0.5) { // Threshold for relevance
matches.push({ pattern, similarity })
}

638
src/patterns/additional-patterns.json
View file

@ -1,638 +0,0 @@
{
"version": "2.0.0",
"description": "Additional patterns to improve coverage to 90%+",
"patterns": [
{
"id": "synonym_find",
"category": "navigational",
"examples": ["find machine learning papers", "locate AI research", "get neural network docs"],
"pattern": "(?:find|locate|get|fetch|retrieve)\\s+(.+)",
"template": { "like": "${1}" },
"confidence": 0.9
},
{
"id": "synonym_show",
"category": "navigational",
"examples": ["show me recent papers", "display all results", "list available options"],
"pattern": "(?:show\\s+me|display|list|view)\\s+(.+)",
"template": { "like": "${1}" },
"confidence": 0.88
},
{
"id": "who_created",
"category": "relational",
"examples": ["who created React", "who wrote this paper", "who invented the internet"],
"pattern": "who\\s+(?:created|wrote|invented|developed|made)\\s+(.+)",
"template": {
"like": "${1}",
"connected": { "relationship": "creator" }
},
"confidence": 0.92
},
{
"id": "when_temporal",
"category": "temporal",
"examples": ["when was Python created", "when did AI start"],
"pattern": "when\\s+(?:was|did|were)\\s+(.+?)\\s+(?:created|started|invented|published)",
"template": {
"like": "${1}",
"where": { "date": { "exists": true } }
},
"confidence": 0.85
},
{
"id": "where_location",
"category": "spatial",
"examples": ["where is Stanford University", "where can I find documentation"],
"pattern": "where\\s+(?:is|are|can\\s+I\\s+find)\\s+(.+)",
"template": {
"like": "${1}",
"where": { "location": { "exists": true } }
},
"confidence": 0.83
},
{
"id": "comparison_vs",
"category": "comparative",
"examples": ["Python vs JavaScript", "React vs Vue", "TensorFlow vs PyTorch"],
"pattern": "(.+?)\\s+vs\\.?\\s+(.+)",
"template": {
"like": "${1} ${2}",
"boost": "comparison"
},
"confidence": 0.9
},
{
"id": "difference_between",
"category": "comparative",
"examples": ["difference between AI and ML", "what's the difference between React and Angular"],
"pattern": "(?:difference|differences)\\s+between\\s+(.+?)\\s+and\\s+(.+)",
"template": {
"like": "${1} ${2} comparison",
"boost": "comparison"
},
"confidence": 0.88
},
{
"id": "similar_to",
"category": "relational",
"examples": ["similar to Python", "papers like this one", "alternatives to React"],
"pattern": "(?:similar\\s+to|like|alternatives?\\s+to)\\s+(.+)",
"template": {
"like": "${1}",
"boost": "similarity"
},
"confidence": 0.87
},
{
"id": "action_download",
"category": "transactional",
"examples": ["download Python", "download the dataset", "get the PDF"],
"pattern": "(?:download|get|fetch)\\s+(?:the\\s+)?(.+?)\\s*(?:pdf|file|document|dataset)?",
"template": {
"like": "${1}",
"where": { "downloadable": true }
},
"confidence": 0.85
},
{
"id": "action_create",
"category": "transactional",
"examples": ["create new project", "make a new document", "generate report"],
"pattern": "(?:create|make|generate|build)\\s+(?:new\\s+)?(.+)",
"template": {
"like": "${1} template",
"boost": "tutorial"
},
"confidence": 0.82
},
{
"id": "how_many",
"category": "aggregation",
"examples": ["how many papers about AI", "count of documents"],
"pattern": "(?:how\\s+many|count\\s+of|number\\s+of)\\s+(.+)",
"template": {
"like": "${1}",
"aggregate": "count"
},
"confidence": 0.9
},
{
"id": "average_of",
"category": "aggregation",
"examples": ["average citations", "mean score", "median value"],
"pattern": "(?:average|mean|median)\\s+(?:of\\s+)?(.+)",
"template": {
"like": "${1}",
"aggregate": "average"
},
"confidence": 0.85
},
{
"id": "tutorial_howto",
"category": "informational",
"examples": ["tutorial on machine learning", "guide to Python", "how to use React"],
"pattern": "(?:tutorial|guide|how\\s+to\\s+use)\\s+(?:on|to|for)?\\s*(.+)",
"template": {
"like": "${1} tutorial",
"boost": "educational"
},
"confidence": 0.92
},
{
"id": "documentation_for",
"category": "informational",
"examples": ["documentation for React", "docs on Python", "API reference"],
"pattern": "(?:documentation|docs|reference|manual)\\s+(?:for|on|about)?\\s*(.+)",
"template": {
"like": "${1} documentation",
"where": { "type": "documentation" }
},
"confidence": 0.93
},
{
"id": "research_on",
"category": "academic",
"examples": ["research on AI safety", "papers about climate change", "studies on COVID"],
"pattern": "(?:research|papers?|studies)\\s+(?:on|about)\\s+(.+)",
"template": {
"like": "${1}",
"where": { "type": "academic" }
},
"confidence": 0.91
},
{
"id": "latest_newest",
"category": "temporal",
"examples": ["latest research", "newest papers", "most recent updates"],
"pattern": "(?:latest|newest|most\\s+recent|current)\\s+(.+)",
"template": {
"like": "${1}",
"boost": "recent"
},
"confidence": 0.89
},
{
"id": "last_period",
"category": "temporal",
"examples": ["last week", "past month", "previous year"],
"pattern": "(?:last|past|previous)\\s+(week|month|year|day)",
"template": {
"where": {
"date": {
"after": "${1}_ago"
}
}
},
"confidence": 0.87
},
{
"id": "between_dates",
"category": "temporal",
"examples": ["between 2020 and 2023", "from January to March"],
"pattern": "between\\s+(\\d{4}|\\w+)\\s+(?:and|to)\\s+(\\d{4}|\\w+)",
"template": {
"where": {
"date": {
"between": ["${1}", "${2}"]
}
}
},
"confidence": 0.86
},
{
"id": "conversational_need",
"category": "conversational",
"examples": ["I need help with Python", "I want to learn React", "I'm looking for AI papers"],
"pattern": "(?:I\\s+need|I\\s+want|I'm\\s+looking\\s+for)\\s+(.+)",
"template": {
"like": "${1}"
},
"confidence": 0.85
},
{
"id": "conversational_can_you",
"category": "conversational",
"examples": ["can you find papers", "could you show me", "would you search for"],
"pattern": "(?:can|could|would)\\s+you\\s+(?:find|show|search|get)\\s+(?:me\\s+)?(.+)",
"template": {
"like": "${1}"
},
"confidence": 0.84
},
{
"id": "tell_me_about",
"category": "informational",
"examples": ["tell me about machine learning", "explain neural networks"],
"pattern": "(?:tell\\s+me\\s+about|explain|describe)\\s+(.+)",
"template": {
"like": "${1}",
"boost": "educational"
},
"confidence": 0.88
},
{
"id": "is_there_any",
"category": "existence",
"examples": ["is there any research on", "are there any papers about"],
"pattern": "(?:is|are)\\s+there\\s+(?:any\\s+)?(.+?)\\s+(?:on|about|for)\\s+(.+)",
"template": {
"like": "${2} ${1}"
},
"confidence": 0.83
},
{
"id": "with_without",
"category": "filtering",
"examples": ["papers with citations", "results without errors", "documents with images"],
"pattern": "(.+?)\\s+(with|without)\\s+(.+)",
"template": {
"like": "${1}",
"where": {
"${3}": { "exists": true }
}
},
"confidence": 0.85
},
{
"id": "and_but_not",
"category": "combined",
"examples": ["AI and ML but not deep learning", "Python and Django but not Flask"],
"pattern": "(.+?)\\s+and\\s+(.+?)\\s+but\\s+not\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": {
"not": "${3}"
}
},
"confidence": 0.82
},
{
"id": "all_that_have",
"category": "filtering",
"examples": ["all papers that have citations", "all documents that contain"],
"pattern": "all\\s+(.+?)\\s+that\\s+(?:have|contain|include)\\s+(.+)",
"template": {
"like": "${1}",
"where": {
"${2}": { "exists": true }
}
},
"confidence": 0.86
},
{
"id": "starting_with",
"category": "filtering",
"examples": ["starting with A", "beginning with chapter", "ending with PDF"],
"pattern": "(.+?)\\s+(?:starting|beginning|ending)\\s+with\\s+(.+)",
"template": {
"like": "${1}",
"where": {
"pattern": "${2}"
}
},
"confidence": 0.84
},
{
"id": "source_code",
"category": "technical",
"examples": ["source code for React", "GitHub repository", "code examples"],
"pattern": "(?:source\\s+code|github|repository|code\\s+examples?)\\s+(?:for|of)?\\s*(.+)",
"template": {
"like": "${1} code",
"where": { "type": "code" }
},
"confidence": 0.87
},
{
"id": "api_endpoint",
"category": "technical",
"examples": ["API endpoint for users", "REST API documentation", "GraphQL schema"],
"pattern": "(?:API|REST|GraphQL)\\s+(?:endpoint|documentation|schema)\\s+(?:for)?\\s*(.+)",
"template": {
"like": "${1} API",
"where": { "type": "api" }
},
"confidence": 0.89
},
{
"id": "example_of",
"category": "informational",
"examples": ["example of machine learning", "sample code", "demo application"],
"pattern": "(?:example|sample|demo)\\s+(?:of|for)?\\s*(.+)",
"template": {
"like": "${1} example",
"boost": "educational"
},
"confidence": 0.86
},
{
"id": "definition_of",
"category": "informational",
"examples": ["definition of AI", "what does ML mean", "meaning of neural network"],
"pattern": "(?:definition\\s+of|what\\s+does\\s+(.+?)\\s+mean|meaning\\s+of)\\s*(.+)",
"template": {
"like": "${1}${2} definition",
"boost": "educational"
},
"confidence": 0.91
},
{
"id": "benchmark_performance",
"category": "comparative",
"examples": ["benchmark results", "performance comparison", "speed test"],
"pattern": "(?:benchmark|performance|speed\\s+test)\\s+(?:results?|comparison)?\\s*(?:for|of)?\\s*(.+)",
"template": {
"like": "${1} benchmark",
"where": { "type": "benchmark" }
},
"confidence": 0.85
},
{
"id": "price_cost",
"category": "commercial",
"examples": ["price of AWS", "cost of hosting", "pricing for services"],
"pattern": "(?:price|cost|pricing)\\s+(?:of|for)\\s+(.+)",
"template": {
"like": "${1} pricing",
"where": { "type": "commercial" }
},
"confidence": 0.88
},
{
"id": "free_open_source",
"category": "commercial",
"examples": ["free alternatives to", "open source version", "free tools for"],
"pattern": "(?:free|open\\s+source)\\s+(?:alternatives?\\s+to|version\\s+of|tools?\\s+for)\\s+(.+)",
"template": {
"like": "${1}",
"where": { "license": "free" }
},
"confidence": 0.87
},
{
"id": "step_by_step",
"category": "informational",
"examples": ["step by step guide", "walkthrough", "detailed instructions"],
"pattern": "(?:step\\s+by\\s+step|walkthrough|detailed\\s+instructions)\\s+(?:for|on)?\\s*(.+)",
"template": {
"like": "${1} tutorial detailed",
"boost": "educational"
},
"confidence": 0.9
},
{
"id": "pros_cons",
"category": "comparative",
"examples": ["pros and cons of React", "advantages of Python", "benefits of AI"],
"pattern": "(?:pros\\s+and\\s+cons|advantages?|benefits?|disadvantages?)\\s+(?:of|for)\\s+(.+)",
"template": {
"like": "${1} analysis",
"boost": "comparison"
},
"confidence": 0.86
},
{
"id": "use_cases",
"category": "informational",
"examples": ["use cases for blockchain", "applications of AI", "when to use React"],
"pattern": "(?:use\\s+cases?|applications?|when\\s+to\\s+use)\\s+(?:for|of)?\\s*(.+)",
"template": {
"like": "${1} applications",
"boost": "practical"
},
"confidence": 0.88
},
{
"id": "troubleshoot_fix",
"category": "technical",
"examples": ["troubleshoot Python error", "fix React issue", "solve problem with"],
"pattern": "(?:troubleshoot|fix|solve|debug|resolve)\\s+(.+?)\\s*(?:error|issue|problem|bug)?",
"template": {
"like": "${1} solution",
"where": { "type": "troubleshooting" }
},
"confidence": 0.87
},
{
"id": "compatible_with",
"category": "technical",
"examples": ["compatible with Python 3", "works with React", "supports Windows"],
"pattern": "(?:compatible\\s+with|works\\s+with|supports)\\s+(.+)",
"template": {
"like": "${1} compatibility",
"where": { "compatibility": "${1}" }
},
"confidence": 0.85
},
{
"id": "version_specific",
"category": "technical",
"examples": ["React version 18", "Python 3.11", "Node.js v20"],
"pattern": "(.+?)\\s+version\\s+(\\d+(?:\\.\\d+)*)",
"template": {
"like": "${1}",
"where": { "version": "${2}" }
},
"confidence": 0.9
},
{
"id": "cheat_sheet",
"category": "informational",
"examples": ["cheat sheet for Python", "quick reference", "cheatsheet React"],
"pattern": "(?:cheat\\s*sheet|quick\\s+reference|reference\\s+card)\\s+(?:for)?\\s*(.+)",
"template": {
"like": "${1} cheatsheet",
"boost": "educational"
},
"confidence": 0.91
},
{
"id": "getting_started",
"category": "informational",
"examples": ["getting started with React", "introduction to Python", "beginner guide"],
"pattern": "(?:getting\\s+started|introduction|beginner'?s?\\s+guide)\\s+(?:with|to|for)?\\s*(.+)",
"template": {
"like": "${1} beginner",
"boost": "educational"
},
"confidence": 0.92
},
{
"id": "advanced_expert",
"category": "informational",
"examples": ["advanced Python techniques", "expert guide", "pro tips for"],
"pattern": "(?:advanced|expert|pro\\s+tips?)\\s+(?:techniques?|guide)?\\s*(?:for|on)?\\s*(.+)",
"template": {
"like": "${1} advanced",
"boost": "expert"
},
"confidence": 0.87
},
{
"id": "list_of_all",
"category": "aggregation",
"examples": ["list of all features", "all available options", "complete list"],
"pattern": "(?:list\\s+of\\s+all|all\\s+available|complete\\s+list)\\s+(.+)",
"template": {
"like": "${1}",
"limit": 1000
},
"confidence": 0.88
},
{
"id": "trending_popular",
"category": "temporal",
"examples": ["trending topics", "popular papers", "hot discussions"],
"pattern": "(?:trending|popular|hot|viral)\\s+(.+)",
"template": {
"like": "${1}",
"boost": "popular"
},
"confidence": 0.86
},
{
"id": "under_development",
"category": "temporal",
"examples": ["under development", "coming soon", "in progress"],
"pattern": "(?:under\\s+development|coming\\s+soon|in\\s+progress|upcoming)\\s*(.+)?",
"template": {
"like": "${1}",
"where": { "status": "development" }
},
"confidence": 0.84
},
{
"id": "deprecated_obsolete",
"category": "temporal",
"examples": ["deprecated features", "obsolete methods", "legacy code"],
"pattern": "(?:deprecated|obsolete|legacy|outdated)\\s+(.+)",
"template": {
"like": "${1}",
"where": { "status": "deprecated" }
},
"confidence": 0.85
},
{
"id": "migration_upgrade",
"category": "technical",
"examples": ["migrate from React 17 to 18", "upgrade guide", "migration path"],
"pattern": "(?:migrate|upgrade|migration\\s+path)\\s+(?:from\\s+)?(.+?)\\s+(?:to\\s+(.+))?",
"template": {
"like": "${1} ${2} migration",
"where": { "type": "migration" }
},
"confidence": 0.86
},
{
"id": "industry_sector",
"category": "domain",
"examples": ["fintech applications", "healthcare AI", "education technology"],
"pattern": "(?:fintech|healthcare|education|finance|medical|legal|retail)\\s+(.+)",
"template": {
"like": "${1}",
"where": { "industry": "${0}" }
},
"confidence": 0.85
},
{
"id": "security_vulnerability",
"category": "technical",
"examples": ["security issues", "vulnerability in", "CVE for"],
"pattern": "(?:security|vulnerability|CVE)\\s+(?:issues?|in|for)?\\s*(.+)",
"template": {
"like": "${1} security",
"where": { "type": "security" }
},
"confidence": 0.89
},
{
"id": "performance_optimization",
"category": "technical",
"examples": ["optimize React performance", "speed up Python", "improve efficiency"],
"pattern": "(?:optimize|speed\\s+up|improve\\s+efficiency)\\s+(?:of)?\\s*(.+?)\\s*(?:performance)?",
"template": {
"like": "${1} optimization",
"boost": "performance"
},
"confidence": 0.87
},
{
"id": "integration_with",
"category": "technical",
"examples": ["integrate React with Redux", "connect Python to database"],
"pattern": "(?:integrate|connect|interface)\\s+(.+?)\\s+(?:with|to)\\s+(.+)",
"template": {
"like": "${1} ${2} integration",
"where": { "type": "integration" }
},
"confidence": 0.86
},
{
"id": "alternative_instead",
"category": "comparative",
"examples": ["instead of React", "alternative to Python", "replacement for"],
"pattern": "(?:instead\\s+of|alternative\\s+to|replacement\\s+for|substitute\\s+for)\\s+(.+)",
"template": {
"like": "${1} alternative",
"boost": "comparison"
},
"confidence": 0.88
},
{
"id": "based_on",
"category": "relational",
"examples": ["based on React", "built with Python", "powered by"],
"pattern": "(?:based\\s+on|built\\s+with|powered\\s+by|using)\\s+(.+)",
"template": {
"like": "${1}",
"connected": { "technology": "${1}" }
},
"confidence": 0.85
},
{
"id": "requires_needs",
"category": "technical",
"examples": ["requires Python 3", "needs Node.js", "dependencies for"],
"pattern": "(?:requires?|needs?|dependencies\\s+for)\\s+(.+)",
"template": {
"like": "${1} requirements",
"where": { "requirements": "${1}" }
},
"confidence": 0.86
},
{
"id": "official_docs",
"category": "navigational",
"examples": ["official React documentation", "official Python site"],
"pattern": "official\\s+(.+?)\\s*(?:documentation|docs|site|website)?",
"template": {
"like": "${1} official",
"where": { "official": true }
},
"confidence": 0.93
},
{
"id": "community_forum",
"category": "navigational",
"examples": ["React community", "Python forum", "Discord server for"],
"pattern": "(?:community|forum|discord|slack|discussion)\\s+(?:for)?\\s*(.+)",
"template": {
"like": "${1} community",
"where": { "type": "community" }
},
"confidence": 0.84
},
{
"id": "roadmap_timeline",
"category": "temporal",
"examples": ["roadmap for React", "timeline of AI development", "history of Python"],
"pattern": "(?:roadmap|timeline|history)\\s+(?:for|of)\\s+(.+)",
"template": {
"like": "${1} roadmap",
"boost": "timeline"
},
"confidence": 0.85
}
]
}

2036
src/patterns/comprehensive-library.json
View file

File diff suppressed because it is too large Load diff

539
src/patterns/domain-patterns.json
View file

@ -1,539 +0,0 @@
{
"version": "2.0.0",
"description": "Domain-specific patterns for 95%+ coverage",
"patterns": [
{
"id": "medical_symptoms",
"category": "domain_specific",
"domain": "medical",
"examples": ["symptoms of COVID", "symptoms of diabetes", "signs of heart attack"],
"pattern": "(?:symptoms?|signs?)\\s+(?:of|for)\\s+(.+)",
"template": {
"like": "${1} symptoms",
"where": { "domain": "medical", "type": "symptoms" }
},
"confidence": 0.95,
"frequency": "high"
},
{
"id": "medical_side_effects",
"category": "domain_specific",
"domain": "medical",
"examples": ["aspirin side effects", "vaccine side effects", "metformin side effects"],
"pattern": "(.+?)\\s+side\\s+effects?",
"template": {
"like": "${1} side effects",
"where": { "domain": "medical", "type": "medication" }
},
"confidence": 0.94,
"frequency": "high"
},
{
"id": "medical_treatment",
"category": "domain_specific",
"domain": "medical",
"examples": ["treatment for depression", "cure for cancer", "therapy for anxiety"],
"pattern": "(?:treatment|cure|therapy|remedy)\\s+(?:for|of)\\s+(.+)",
"template": {
"like": "${1} treatment",
"where": { "domain": "medical", "type": "treatment" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "medical_pain",
"category": "domain_specific",
"domain": "medical",
"examples": ["chest pain causes", "back pain relief", "headache remedies"],
"pattern": "(.+?)\\s+pain\\s+(?:causes?|relief|remedies?)",
"template": {
"like": "${1} pain",
"where": { "domain": "medical", "type": "pain" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "medical_test_results",
"category": "domain_specific",
"domain": "medical",
"examples": ["blood test results", "MRI results meaning", "normal cholesterol levels"],
"pattern": "(?:(.+?)\\s+)?(?:test\\s+)?results?\\s+(?:meaning|interpretation|normal\\s+range)",
"template": {
"like": "${1} test results",
"where": { "domain": "medical", "type": "diagnostic" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "legal_definition",
"category": "domain_specific",
"domain": "legal",
"examples": ["tort law definition", "what is habeas corpus", "felony vs misdemeanor"],
"pattern": "(?:what\\s+is\\s+)?(.+?)\\s+(?:definition|meaning|vs\\.?\\s+(.+))",
"template": {
"like": "${1} ${2} legal definition",
"where": { "domain": "legal", "type": "definition" }
},
"confidence": 0.90,
"frequency": "high"
},
{
"id": "legal_how_to_file",
"category": "domain_specific",
"domain": "legal",
"examples": ["how to file bankruptcy", "file for divorce", "file a complaint"],
"pattern": "(?:how\\s+to\\s+)?file\\s+(?:for\\s+)?(.+)",
"template": {
"like": "file ${1} procedure",
"where": { "domain": "legal", "type": "filing" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "legal_jurisdiction",
"category": "domain_specific",
"domain": "legal",
"examples": ["marijuana laws in California", "gun laws by state", "divorce laws in Texas"],
"pattern": "(.+?)\\s+laws?\\s+(?:in|by)\\s+(.+)",
"template": {
"like": "${1} laws ${2}",
"where": { "domain": "legal", "jurisdiction": "${2}" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "legal_statute_limitations",
"category": "domain_specific",
"domain": "legal",
"examples": ["statute of limitations personal injury", "SOL for debt collection"],
"pattern": "statute\\s+of\\s+limitations?\\s+(?:for\\s+)?(.+)",
"template": {
"like": "${1} statute of limitations",
"where": { "domain": "legal", "type": "statute" }
},
"confidence": 0.93,
"frequency": "medium"
},
{
"id": "financial_stock_price",
"category": "domain_specific",
"domain": "financial",
"examples": ["AAPL stock price", "Tesla share price", "GOOGL quote"],
"pattern": "([A-Z]{1,5})\\s+(?:stock\\s+)?(?:price|quote|shares?)",
"template": {
"like": "${1} stock price",
"where": { "domain": "financial", "type": "stock", "ticker": "${1}" }
},
"confidence": 0.95,
"frequency": "high"
},
{
"id": "financial_interest_rates",
"category": "domain_specific",
"domain": "financial",
"examples": ["mortgage interest rates", "CD rates today", "Fed interest rate"],
"pattern": "(.+?)\\s+(?:interest\\s+)?rates?\\s*(?:today|current)?",
"template": {
"like": "${1} interest rates",
"where": { "domain": "financial", "type": "rates" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "financial_calculator",
"category": "domain_specific",
"domain": "financial",
"examples": ["mortgage calculator", "loan payment calculator", "retirement calculator"],
"pattern": "(.+?)\\s+calculator",
"template": {
"like": "${1} calculator",
"where": { "domain": "financial", "type": "calculator" }
},
"confidence": 0.94,
"frequency": "high"
},
{
"id": "financial_tax",
"category": "domain_specific",
"domain": "financial",
"examples": ["tax deductions for homeowners", "capital gains tax rate", "tax brackets 2024"],
"pattern": "tax\\s+(.+?)\\s+(?:for\\s+(.+)|rate|brackets?)",
"template": {
"like": "tax ${1} ${2}",
"where": { "domain": "financial", "type": "tax" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "financial_credit_score",
"category": "domain_specific",
"domain": "financial",
"examples": ["credit score for mortgage", "improve credit score", "free credit report"],
"pattern": "(?:credit\\s+score|credit\\s+report)\\s+(?:for\\s+)?(.+)?",
"template": {
"like": "credit score ${1}",
"where": { "domain": "financial", "type": "credit" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "academic_citation",
"category": "domain_specific",
"domain": "academic",
"examples": ["cite website APA", "MLA citation format", "Chicago style bibliography"],
"pattern": "(?:cite|citation)\\s+(.+?)\\s+(?:in\\s+)?(APA|MLA|Chicago|Harvard)",
"template": {
"like": "${1} citation ${2}",
"where": { "domain": "academic", "type": "citation", "style": "${2}" }
},
"confidence": 0.94,
"frequency": "high"
},
{
"id": "academic_publications",
"category": "domain_specific",
"domain": "academic",
"examples": ["Einstein publications", "papers by Hinton", "Smith et al 2023"],
"pattern": "(?:(.+?)\\s+publications?|papers?\\s+by\\s+(.+))",
"template": {
"like": "${1}${2} publications",
"where": { "domain": "academic", "type": "publication" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "academic_peer_reviewed",
"category": "domain_specific",
"domain": "academic",
"examples": ["peer reviewed articles on climate change", "scholarly articles about AI"],
"pattern": "(?:peer\\s+reviewed|scholarly)\\s+(?:articles?|papers?)\\s+(?:on|about)\\s+(.+)",
"template": {
"like": "${1} peer reviewed",
"where": { "domain": "academic", "type": "peer_reviewed" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "academic_journal_impact",
"category": "domain_specific",
"domain": "academic",
"examples": ["Nature impact factor", "Science journal ranking", "PNAS impact factor"],
"pattern": "(.+?)\\s+(?:impact\\s+factor|journal\\s+ranking)",
"template": {
"like": "${1} impact factor",
"where": { "domain": "academic", "type": "journal" }
},
"confidence": 0.92,
"frequency": "medium"
},
{
"id": "ecommerce_reviews",
"category": "domain_specific",
"domain": "ecommerce",
"examples": ["iPhone 15 reviews", "best laptop reviews", "Samsung TV ratings"],
"pattern": "(.+?)\\s+(?:reviews?|ratings?)",
"template": {
"like": "${1} reviews",
"where": { "domain": "ecommerce", "type": "review" }
},
"confidence": 0.95,
"frequency": "high"
},
{
"id": "ecommerce_price_comparison",
"category": "domain_specific",
"domain": "ecommerce",
"examples": ["cheapest PS5", "best price MacBook", "lowest price Nike shoes"],
"pattern": "(?:cheapest|best\\s+price|lowest\\s+price)\\s+(.+)",
"template": {
"like": "${1} price",
"where": { "domain": "ecommerce", "sort": "price_asc" }
},
"confidence": 0.94,
"frequency": "high"
},
{
"id": "ecommerce_deals",
"category": "domain_specific",
"domain": "ecommerce",
"examples": ["Amazon deals today", "Black Friday sales", "coupon codes Target"],
"pattern": "(.+?)\\s+(?:deals?|sales?|coupon\\s+codes?)\\s*(?:today)?",
"template": {
"like": "${1} deals",
"where": { "domain": "ecommerce", "type": "deals" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "ecommerce_in_stock",
"category": "domain_specific",
"domain": "ecommerce",
"examples": ["PS5 in stock", "RTX 4090 availability", "iPhone 15 where to buy"],
"pattern": "(.+?)\\s+(?:in\\s+stock|availability|where\\s+to\\s+buy)",
"template": {
"like": "${1} availability",
"where": { "domain": "ecommerce", "in_stock": true }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "ecommerce_size_chart",
"category": "domain_specific",
"domain": "ecommerce",
"examples": ["Nike size chart", "ring size guide", "clothing size conversion"],
"pattern": "(.+?)\\s+size\\s+(?:chart|guide|conversion)",
"template": {
"like": "${1} size chart",
"where": { "domain": "ecommerce", "type": "sizing" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "technical_error_fix",
"category": "domain_specific",
"domain": "technical",
"examples": ["error 404 fix", "blue screen of death", "kernel panic solution"],
"pattern": "(?:error\\s+)?(.+?)\\s+(?:fix|solution|resolve)",
"template": {
"like": "${1} fix",
"where": { "domain": "technical", "type": "troubleshooting" }
},
"confidence": 0.94,
"frequency": "high"
},
{
"id": "technical_not_working",
"category": "domain_specific",
"domain": "technical",
"examples": ["WiFi not working", "printer not responding", "app won't open"],
"pattern": "(.+?)\\s+(?:not\\s+working|won't\\s+(?:open|start|load)|not\\s+responding)",
"template": {
"like": "${1} troubleshooting",
"where": { "domain": "technical", "type": "issue" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "technical_how_to_reset",
"category": "domain_specific",
"domain": "technical",
"examples": ["reset iPhone", "factory reset laptop", "reset password Windows"],
"pattern": "(?:how\\s+to\\s+)?(?:reset|factory\\s+reset)\\s+(.+)",
"template": {
"like": "reset ${1}",
"where": { "domain": "technical", "type": "reset" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "technical_update",
"category": "domain_specific",
"domain": "technical",
"examples": ["update Windows 11", "iOS 17 update", "Chrome latest version"],
"pattern": "(?:update|upgrade)\\s+(.+?)\\s*(?:to\\s+(.+))?",
"template": {
"like": "${1} update ${2}",
"where": { "domain": "technical", "type": "update" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "technical_driver_download",
"category": "domain_specific",
"domain": "technical",
"examples": ["NVIDIA driver download", "printer driver HP", "Realtek audio driver"],
"pattern": "(.+?)\\s+driver\\s*(?:download)?",
"template": {
"like": "${1} driver",
"where": { "domain": "technical", "type": "driver" }
},
"confidence": 0.93,
"frequency": "medium"
},
{
"id": "technical_speed_up",
"category": "domain_specific",
"domain": "technical",
"examples": ["speed up computer", "make phone faster", "optimize Windows"],
"pattern": "(?:speed\\s+up|make\\s+(.+?)\\s+faster|optimize)\\s+(.+)",
"template": {
"like": "${1}${2} optimization",
"where": { "domain": "technical", "type": "performance" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "medical_doctor_specialist",
"category": "domain_specific",
"domain": "medical",
"examples": ["cardiologist near me", "best dermatologist", "pediatrician reviews"],
"pattern": "(.+?(?:ologist|ician|doctor))\\s+(?:near\\s+me|reviews?|best)",
"template": {
"like": "${1}",
"where": { "domain": "medical", "type": "specialist" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "medical_vaccine",
"category": "domain_specific",
"domain": "medical",
"examples": ["COVID vaccine side effects", "flu shot effectiveness", "vaccine schedule babies"],
"pattern": "(.+?)\\s+vaccine\\s+(.+)",
"template": {
"like": "${1} vaccine ${2}",
"where": { "domain": "medical", "type": "vaccine" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "legal_lawyer_type",
"category": "domain_specific",
"domain": "legal",
"examples": ["divorce lawyer near me", "personal injury attorney", "criminal defense lawyer"],
"pattern": "(.+?)\\s+(?:lawyer|attorney)\\s*(?:near\\s+me)?",
"template": {
"like": "${1} lawyer",
"where": { "domain": "legal", "type": "attorney" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "legal_contract_template",
"category": "domain_specific",
"domain": "legal",
"examples": ["rental agreement template", "NDA template", "employment contract sample"],
"pattern": "(.+?)\\s+(?:template|sample|form)",
"template": {
"like": "${1} template",
"where": { "domain": "legal", "type": "template" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "financial_crypto",
"category": "domain_specific",
"domain": "financial",
"examples": ["Bitcoin price", "Ethereum forecast", "buy cryptocurrency"],
"pattern": "(.+?)\\s+(?:price|forecast|buy|sell)",
"template": {
"like": "${1} cryptocurrency",
"where": { "domain": "financial", "type": "crypto" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "financial_investment_strategy",
"category": "domain_specific",
"domain": "financial",
"examples": ["401k investment strategy", "best ETFs 2024", "dividend investing"],
"pattern": "(.+?)\\s+(?:investment\\s+strategy|investing)",
"template": {
"like": "${1} investment strategy",
"where": { "domain": "financial", "type": "investment" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "academic_research_methodology",
"category": "domain_specific",
"domain": "academic",
"examples": ["qualitative research methods", "sample size calculation", "statistical analysis"],
"pattern": "(.+?)\\s+(?:research\\s+methods?|methodology)",
"template": {
"like": "${1} methodology",
"where": { "domain": "academic", "type": "methodology" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "academic_grant_funding",
"category": "domain_specific",
"domain": "academic",
"examples": ["NSF grant opportunities", "research funding biology", "PhD funding"],
"pattern": "(.+?)\\s+(?:grant|funding)\\s*(?:opportunities)?",
"template": {
"like": "${1} funding",
"where": { "domain": "academic", "type": "funding" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "ecommerce_return_policy",
"category": "domain_specific",
"domain": "ecommerce",
"examples": ["Amazon return policy", "Walmart refund", "exchange policy Best Buy"],
"pattern": "(.+?)\\s+(?:return\\s+policy|refund|exchange)",
"template": {
"like": "${1} return policy",
"where": { "domain": "ecommerce", "type": "policy" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "ecommerce_warranty",
"category": "domain_specific",
"domain": "ecommerce",
"examples": ["Apple warranty check", "extended warranty worth it", "warranty claim Samsung"],
"pattern": "(.+?)\\s+warranty\\s*(.+)?",
"template": {
"like": "${1} warranty ${2}",
"where": { "domain": "ecommerce", "type": "warranty" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "technical_backup_restore",
"category": "domain_specific",
"domain": "technical",
"examples": ["backup iPhone", "restore from backup", "cloud backup options"],
"pattern": "(?:backup|restore)\\s+(?:from\\s+)?(.+)",
"template": {
"like": "${1} backup",
"where": { "domain": "technical", "type": "backup" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "technical_compatibility",
"category": "domain_specific",
"domain": "technical",
"examples": ["compatible with Windows 11", "works with Mac", "supports Android"],
"pattern": "(?:compatible\\s+with|works\\s+with|supports?)\\s+(.+)",
"template": {
"like": "${1} compatibility",
"where": { "domain": "technical", "type": "compatibility" }
},
"confidence": 0.91,
"frequency": "medium"
}
]
}

715
src/patterns/library.json
View file

@ -1,715 +0,0 @@
{
"version": "2.0.0",
"patterns": [
{
"id": "info_what",
"category": "informational",
"examples": ["what is machine learning", "what are neural networks", "what does AI mean"],
"pattern": "what (is|are|does) (.+)",
"template": {
"like": "${2}"
},
"confidence": 0.9
},
{
"id": "info_how",
"category": "informational",
"examples": ["how to train a model", "how does clustering work", "how to implement search"],
"pattern": "how (to|does|do|can) (.+)",
"template": {
"like": "${2}",
"where": { "type": "tutorial" }
},
"confidence": 0.85
},
{
"id": "info_why",
"category": "informational",
"examples": ["why use vector databases", "why does overfitting occur"],
"pattern": "why (does|do|is|are|use) (.+)",
"template": {
"like": "${2}",
"where": { "type": "explanation" }
},
"confidence": 0.85
},
{
"id": "info_when",
"category": "informational",
"examples": ["when did deep learning start", "when was transformer invented"],
"pattern": "when (did|was|were|is|are) (.+)",
"template": {
"like": "${2}",
"where": { "type": "event" }
},
"confidence": 0.85
},
{
"id": "info_where",
"category": "informational",
"examples": ["where is Stanford located", "where can I find datasets"],
"pattern": "where (is|are|can|do) (.+)",
"template": {
"like": "${2}",
"where": { "type": "location" }
},
"confidence": 0.85
},
{
"id": "info_who",
"category": "informational",
"examples": ["who invented transformers", "who is Geoffrey Hinton"],
"pattern": "who (is|are|invented|created|made) (.+)",
"template": {
"like": "${2}",
"where": { "type": "person" }
},
"confidence": 0.85
},
{
"id": "info_definition",
"category": "informational",
"examples": ["machine learning definition", "AI meaning", "what neural network means"],
"pattern": "(.+) (definition|meaning|means)",
"template": {
"like": "${1}",
"where": { "type": "definition" }
},
"confidence": 0.9
},
{
"id": "info_explain",
"category": "informational",
"examples": ["explain backpropagation", "explain how transformers work"],
"pattern": "explain (.+)",
"template": {
"like": "${1}",
"where": { "type": "explanation" }
},
"confidence": 0.9
},
{
"id": "info_tutorial",
"category": "informational",
"examples": ["tutorial on deep learning", "pytorch tutorial", "guide to NLP"],
"pattern": "(tutorial|guide|course) (on|to|for)? (.+)",
"template": {
"like": "${3}",
"where": { "type": "tutorial" }
},
"confidence": 0.9
},
{
"id": "nav_entity",
"category": "navigational",
"examples": ["OpenAI website", "Google homepage", "GitHub tensorflow"],
"pattern": "([A-Z][\\w]+) (website|homepage|page|site)",
"template": {
"where": { "name": "${1}", "type": "website" }
},
"confidence": 0.95
},
{
"id": "nav_goto",
"category": "navigational",
"examples": ["go to documentation", "navigate to settings"],
"pattern": "(go to|navigate to|open|show) (.+)",
"template": {
"where": { "name": "${2}" }
},
"confidence": 0.85
},
{
"id": "nav_profile",
"category": "navigational",
"examples": ["John Smith profile", "user profile", "my account"],
"pattern": "(.+) (profile|account|page)",
"template": {
"where": { "name": "${1}", "type": "profile" }
},
"confidence": 0.85
},
{
"id": "trans_buy",
"category": "transactional",
"examples": ["buy GPU", "purchase subscription", "order dataset"],
"pattern": "(buy|purchase|order|get) (.+)",
"template": {
"like": "${2}",
"where": { "type": "product", "available": true }
},
"confidence": 0.9
},
{
"id": "trans_download",
"category": "transactional",
"examples": ["download model", "download dataset", "get paper PDF"],
"pattern": "(download|get|fetch) (.+)",
"template": {
"like": "${2}",
"where": { "type": "downloadable" }
},
"confidence": 0.9
},
{
"id": "trans_subscribe",
"category": "transactional",
"examples": ["subscribe to newsletter", "follow updates"],
"pattern": "(subscribe|follow|watch) (to )? (.+)",
"template": {
"like": "${3}",
"where": { "type": "subscription" }
},
"confidence": 0.85
},
{
"id": "commercial_reviews",
"category": "commercial",
"examples": ["tensorflow reviews", "best practices reviews", "model evaluation"],
"pattern": "(.+) (reviews|ratings|feedback|opinions)",
"template": {
"like": "${1}",
"where": { "type": "review" }
},
"confidence": 0.9
},
{
"id": "commercial_best",
"category": "commercial",
"examples": ["best machine learning framework", "top AI models", "best practices"],
"pattern": "(best|top|greatest|finest) (.+)",
"template": {
"like": "${2}",
"boost": "popular"
},
"confidence": 0.9
},
{
"id": "commercial_compare",
"category": "commercial",
"examples": ["tensorflow vs pytorch", "compare BERT and GPT", "GPT-3 compared to GPT-4"],
"pattern": "(.+) (vs|versus|compared to|vs\\.) (.+)",
"template": {
"like": ["${1}", "${3}"],
"where": { "type": "comparison" }
},
"confidence": 0.95
},
{
"id": "commercial_alternatives",
"category": "commercial",
"examples": ["tensorflow alternatives", "options besides OpenAI", "similar to BERT"],
"pattern": "(.+) (alternatives|options|similar to|like)",
"template": {
"like": "${1}",
"where": { "type": "alternative" }
},
"confidence": 0.85
},
{
"id": "commercial_top_n",
"category": "commercial",
"examples": ["top 10 models", "top 5 papers", "best 3 frameworks"],
"pattern": "(top|best) (\\d+) (.+)",
"template": {
"like": "${3}",
"limit": "${2}",
"boost": "popular"
},
"confidence": 0.9
},
{
"id": "commercial_cheapest",
"category": "commercial",
"examples": ["cheapest GPU", "most affordable cloud", "budget options"],
"pattern": "(cheapest|most affordable|budget|lowest price) (.+)",
"template": {
"like": "${2}",
"orderBy": { "price": "asc" }
},
"confidence": 0.85
},
{
"id": "commercial_pricing",
"category": "commercial",
"examples": ["GPU pricing", "cloud costs", "model training costs"],
"pattern": "(.+) (pricing|price|cost|costs|rates)",
"template": {
"like": "${1}",
"where": { "hasField": "price" }
},
"confidence": 0.85
},
{
"id": "temporal_from_year",
"category": "temporal",
"examples": ["papers from 2023", "research from 2022", "models from last year"],
"pattern": "(.+) from (\\d{4})",
"template": {
"like": "${1}",
"where": { "year": "${2}" }
},
"confidence": 0.95
},
{
"id": "temporal_after",
"category": "temporal",
"examples": ["papers after 2020", "research after January", "models after GPT-3"],
"pattern": "(.+) after (.+)",
"template": {
"like": "${1}",
"where": { "date": { "greaterThan": "${2}" } }
},
"confidence": 0.9
},
{
"id": "temporal_before",
"category": "temporal",
"examples": ["papers before 2020", "research before transformer", "models before BERT"],
"pattern": "(.+) before (.+)",
"template": {
"like": "${1}",
"where": { "date": { "lessThan": "${2}" } }
},
"confidence": 0.9
},
{
"id": "temporal_between",
"category": "temporal",
"examples": ["papers between 2020 and 2023", "research from 2021 to 2022"],
"pattern": "(.+) (between|from) (.+) (and|to) (.+)",
"template": {
"like": "${1}",
"where": { "date": { "between": ["${3}", "${5}"] } }
},
"confidence": 0.85
},
{
"id": "temporal_recent",
"category": "temporal",
"examples": ["recent papers", "latest research", "new models"],
"pattern": "(recent|latest|new|newest) (.+)",
"template": {
"like": "${2}",
"boost": "recent"
},
"confidence": 0.9
},
{
"id": "temporal_last_n_days",
"category": "temporal",
"examples": ["papers last 30 days", "research last week", "models last month"],
"pattern": "(.+) last (\\d+) (days|weeks|months|years)",
"template": {
"like": "${1}",
"where": { "date": { "greaterThan": "${2} ${3} ago" } }
},
"confidence": 0.85
},
{
"id": "temporal_this_period",
"category": "temporal",
"examples": ["papers this year", "research this month", "models this week"],
"pattern": "(.+) this (week|month|year|quarter)",
"template": {
"like": "${1}",
"where": { "date": { "greaterThan": "start of ${2}" } }
},
"confidence": 0.9
},
{
"id": "spatial_near",
"category": "spatial",
"examples": ["conferences near Boston", "labs near Stanford", "companies near me"],
"pattern": "(.+) near (.+)",
"template": {
"like": "${1}",
"where": { "location": { "near": "${2}" } }
},
"confidence": 0.85
},
{
"id": "spatial_in",
"category": "spatial",
"examples": ["companies in Silicon Valley", "universities in Boston", "labs in California"],
"pattern": "(.+) in ([A-Z][\\w\\s]+)",
"template": {
"like": "${1}",
"where": { "location": "${2}" }
},
"confidence": 0.9
},
{
"id": "spatial_at",
"category": "spatial",
"examples": ["researchers at MIT", "papers at conference", "work at Google"],
"pattern": "(.+) at ([A-Z][\\w]+)",
"template": {
"like": "${1}",
"where": { "organization": "${2}" }
},
"confidence": 0.85
},
{
"id": "relational_by_author",
"category": "relational",
"examples": ["papers by Hinton", "research by OpenAI", "models by Google"],
"pattern": "(.+) by ([A-Z][\\w\\s]+)",
"template": {
"like": "${1}",
"connected": { "from": "${2}" }
},
"confidence": 0.95
},
{
"id": "relational_from_source",
"category": "relational",
"examples": ["papers from Stanford", "datasets from Google", "models from OpenAI"],
"pattern": "(.+) from ([A-Z][\\w\\s]+)",
"template": {
"like": "${1}",
"connected": { "from": "${2}" }
},
"confidence": 0.9
},
{
"id": "relational_related",
"category": "relational",
"examples": ["papers related to transformers", "research connected to NLP"],
"pattern": "(.+) (related to|connected to|associated with) (.+)",
"template": {
"like": "${1}",
"connected": { "to": "${3}" }
},
"confidence": 0.85
},
{
"id": "relational_created_by",
"category": "relational",
"examples": ["models created by OpenAI", "datasets created by Google"],
"pattern": "(.+) (created|made|developed|built) by (.+)",
"template": {
"like": "${1}",
"connected": { "from": "${3}", "type": "created" }
},
"confidence": 0.9
},
{
"id": "relational_authored",
"category": "relational",
"examples": ["papers authored by Bengio", "articles written by researchers"],
"pattern": "(.+) (authored|written) by (.+)",
"template": {
"like": "${1}",
"connected": { "from": "${3}", "type": "author" }
},
"confidence": 0.95
},
{
"id": "relational_published",
"category": "relational",
"examples": ["papers published by Nature", "articles published in Science"],
"pattern": "(.+) published (by|in) (.+)",
"template": {
"like": "${1}",
"connected": { "from": "${3}", "type": "publisher" }
},
"confidence": 0.9
},
{
"id": "filter_with",
"category": "filtering",
"examples": ["papers with code", "models with pretrained weights", "datasets with labels"],
"pattern": "(.+) with (.+)",
"template": {
"like": "${1}",
"where": { "${2}": { "exists": true } }
},
"confidence": 0.85
},
{
"id": "filter_without",
"category": "filtering",
"examples": ["papers without code", "models without training", "datasets without labels"],
"pattern": "(.+) without (.+)",
"template": {
"like": "${1}",
"where": { "${2}": { "exists": false } }
},
"confidence": 0.85
},
{
"id": "filter_only",
"category": "filtering",
"examples": ["only open source models", "only free datasets", "papers only"],
"pattern": "(only )? (.+) (only)?",
"template": {
"like": "${2}",
"where": { "exclusive": true }
},
"confidence": 0.75
},
{
"id": "filter_except",
"category": "filtering",
"examples": ["all models except GPT", "papers except reviews", "everything but tutorials"],
"pattern": "(.+) (except|but not|excluding) (.+)",
"template": {
"like": "${1}",
"where": { "notLike": "${3}" }
},
"confidence": 0.85
},
{
"id": "filter_including",
"category": "filtering",
"examples": ["papers including code", "models including documentation"],
"pattern": "(.+) (including|with|containing) (.+)",
"template": {
"like": "${1}",
"where": { "includes": "${3}" }
},
"confidence": 0.85
},
{
"id": "filter_more_than",
"category": "filtering",
"examples": ["papers with more than 100 citations", "models with over 1B parameters"],
"pattern": "(.+) with (more than|over|greater than) (\\d+) (.+)",
"template": {
"like": "${1}",
"where": { "${4}": { "greaterThan": "${3}" } }
},
"confidence": 0.9
},
{
"id": "filter_less_than",
"category": "filtering",
"examples": ["models with less than 1M parameters", "papers with under 10 citations"],
"pattern": "(.+) with (less than|under|fewer than) (\\d+) (.+)",
"template": {
"like": "${1}",
"where": { "${4}": { "lessThan": "${3}" } }
},
"confidence": 0.9
},
{
"id": "filter_exactly",
"category": "filtering",
"examples": ["papers with exactly 5 authors", "models with 12 layers"],
"pattern": "(.+) with (exactly |)(\\d+) (.+)",
"template": {
"like": "${1}",
"where": { "${4}": "${3}" }
},
"confidence": 0.85
},
{
"id": "aggregation_count",
"category": "aggregation",
"examples": ["count papers", "number of models", "how many datasets"],
"pattern": "(count|number of|how many) (.+)",
"template": {
"like": "${2}",
"aggregate": "count"
},
"confidence": 0.9
},
{
"id": "aggregation_average",
"category": "aggregation",
"examples": ["average citations", "mean accuracy", "average performance"],
"pattern": "(average|mean) (.+)",
"template": {
"like": "${2}",
"aggregate": "avg"
},
"confidence": 0.85
},
{
"id": "aggregation_sum",
"category": "aggregation",
"examples": ["total citations", "sum of parameters", "total cost"],
"pattern": "(total|sum of|sum) (.+)",
"template": {
"like": "${2}",
"aggregate": "sum"
},
"confidence": 0.85
},
{
"id": "aggregation_max",
"category": "aggregation",
"examples": ["highest accuracy", "maximum performance", "largest model"],
"pattern": "(highest|maximum|largest|biggest) (.+)",
"template": {
"like": "${2}",
"aggregate": "max"
},
"confidence": 0.85
},
{
"id": "aggregation_min",
"category": "aggregation",
"examples": ["lowest error", "minimum cost", "smallest model"],
"pattern": "(lowest|minimum|smallest|least) (.+)",
"template": {
"like": "${2}",
"aggregate": "min"
},
"confidence": 0.85
},
{
"id": "question_can",
"category": "informational",
"examples": ["can transformers handle images", "can I use this for NLP"],
"pattern": "can (.+)",
"template": {
"like": "${1}",
"where": { "type": "capability" }
},
"confidence": 0.8
},
{
"id": "question_should",
"category": "informational",
"examples": ["should I use tensorflow", "should we implement caching"],
"pattern": "should (I|we|you) (.+)",
"template": {
"like": "${2}",
"where": { "type": "recommendation" }
},
"confidence": 0.8
},
{
"id": "question_which",
"category": "commercial",
"examples": ["which model is best", "which framework to use"],
"pattern": "which (.+)",
"template": {
"like": "${1}",
"where": { "type": "selection" }
},
"confidence": 0.8
},
{
"id": "comparative_better",
"category": "commercial",
"examples": ["is BERT better than GPT", "pytorch better than tensorflow"],
"pattern": "(is )? (.+) better than (.+)",
"template": {
"like": ["${2}", "${3}"],
"where": { "type": "comparison" }
},
"confidence": 0.85
},
{
"id": "comparative_faster",
"category": "commercial",
"examples": ["fastest model", "quickest training", "faster than BERT"],
"pattern": "(fastest|quickest|faster) (.+)",
"template": {
"like": "${2}",
"orderBy": { "speed": "desc" }
},
"confidence": 0.85
},
{
"id": "comparative_more_accurate",
"category": "commercial",
"examples": ["most accurate model", "higher accuracy than"],
"pattern": "(most accurate|highest accuracy|more accurate) (.+)",
"template": {
"like": "${2}",
"orderBy": { "accuracy": "desc" }
},
"confidence": 0.85
},
{
"id": "action_show",
"category": "navigational",
"examples": ["show me papers", "display results", "list models"],
"pattern": "(show|display|list) (me )? (.+)",
"template": {
"like": "${3}"
},
"confidence": 0.85
},
{
"id": "action_find",
"category": "navigational",
"examples": ["find papers about AI", "search for models", "look for datasets"],
"pattern": "(find|search for|look for) (.+)",
"template": {
"like": "${2}"
},
"confidence": 0.9
},
{
"id": "action_get",
"category": "transactional",
"examples": ["get all papers", "fetch datasets", "retrieve models"],
"pattern": "(get|fetch|retrieve) (all )? (.+)",
"template": {
"like": "${3}"
},
"confidence": 0.85
},
{
"id": "combined_complex_1",
"category": "combined",
"examples": ["recent papers by Hinton with more than 50 citations"],
"pattern": "recent (.+) by (.+) with more than (\\d+) (.+)",
"template": {
"like": "${1}",
"connected": { "from": "${2}" },
"where": { "${4}": { "greaterThan": "${3}" } },
"boost": "recent"
},
"confidence": 0.75
},
{
"id": "combined_complex_2",
"category": "combined",
"examples": ["best machine learning papers from 2023 at Stanford"],
"pattern": "best (.+) from (\\d{4}) at (.+)",
"template": {
"like": "${1}",
"where": { "year": "${2}", "organization": "${3}" },
"boost": "popular"
},
"confidence": 0.75
},
{
"id": "combined_complex_3",
"category": "combined",
"examples": ["compare tensorflow and pytorch for computer vision"],
"pattern": "compare (.+) and (.+) for (.+)",
"template": {
"like": ["${1}", "${2}", "${3}"],
"where": { "type": "comparison", "domain": "${3}" }
},
"confidence": 0.75
},
{
"id": "contextual_more_like",
"category": "contextual",
"examples": ["more like this", "similar papers", "find similar"],
"pattern": "(more like|similar to|like) (this|that|these)",
"template": {
"similar": "__context__"
},
"confidence": 0.8
},
{
"id": "contextual_same_but",
"category": "contextual",
"examples": ["same but newer", "same query but from 2023"],
"pattern": "same (query |search |)but (.+)",
"template": {
"__modifier__": "${2}"
},
"confidence": 0.75
}
]
}

266
src/patterns/social-media-patterns.json
View file

@ -1,266 +0,0 @@
{
"version": "2.0.0",
"description": "Social media and content creation patterns",
"patterns": [
{
"id": "social_trending",
"category": "domain_specific",
"domain": "social",
"examples": ["trending on Twitter", "viral TikTok videos", "Instagram trends 2024"],
"pattern": "(?:trending|viral|popular)\\s+(?:on\\s+)?(Twitter|TikTok|Instagram|YouTube|LinkedIn|Reddit|Facebook)\\s*(.+)?",
"template": {
"like": "${1} trending ${2}",
"where": { "domain": "social", "platform": "${1}", "type": "trending" }
},
"confidence": 0.94,
"frequency": "very_high"
},
{
"id": "social_hashtag",
"category": "domain_specific",
"domain": "social",
"examples": ["#AI hashtag", "best hashtags for Instagram", "trending hashtags today"],
"pattern": "(?:#(\\w+)|hashtags?\\s+(?:for\\s+)?(.+))",
"template": {
"like": "hashtag ${1}${2}",
"where": { "domain": "social", "type": "hashtag" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "social_influencer",
"category": "domain_specific",
"domain": "social",
"examples": ["top tech influencers", "Instagram influencers fashion", "YouTube creators gaming"],
"pattern": "(?:top\\s+)?(.+?)\\s+(?:influencers?|creators?|YouTubers?)\\s*(?:on\\s+(.+))?",
"template": {
"like": "${1} influencers ${2}",
"where": { "domain": "social", "type": "influencer" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "social_followers",
"category": "domain_specific",
"domain": "social",
"examples": ["how to get more followers", "increase Instagram followers", "buy Twitter followers"],
"pattern": "(?:how\\s+to\\s+)?(?:get|gain|increase|buy)\\s+(?:more\\s+)?(.+?)\\s+followers?",
"template": {
"like": "${1} followers growth",
"where": { "domain": "social", "type": "growth" }
},
"confidence": 0.93,
"frequency": "very_high"
},
{
"id": "social_content_ideas",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram post ideas", "TikTok video ideas", "LinkedIn content strategy"],
"pattern": "(.+?)\\s+(?:post|video|content|story)\\s+(?:ideas?|strategy|tips?)",
"template": {
"like": "${1} content ideas",
"where": { "domain": "social", "type": "content_strategy" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "social_algorithm",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram algorithm 2024", "TikTok algorithm explained", "YouTube algorithm changes"],
"pattern": "(Instagram|TikTok|YouTube|Twitter|LinkedIn)\\s+algorithm\\s*(.+)?",
"template": {
"like": "${1} algorithm ${2}",
"where": { "domain": "social", "platform": "${1}", "type": "algorithm" }
},
"confidence": 0.94,
"frequency": "high"
},
{
"id": "social_analytics",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram analytics tools", "track Twitter engagement", "social media metrics"],
"pattern": "(.+?)\\s+(?:analytics?|metrics?|insights?|engagement)\\s*(?:tools?)?",
"template": {
"like": "${1} analytics",
"where": { "domain": "social", "type": "analytics" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "social_scheduling",
"category": "domain_specific",
"domain": "social",
"examples": ["best time to post Instagram", "schedule tweets", "social media calendar"],
"pattern": "(?:best\\s+time\\s+to\\s+post|schedule)\\s+(.+)",
"template": {
"like": "${1} posting schedule",
"where": { "domain": "social", "type": "scheduling" }
},
"confidence": 0.90,
"frequency": "high"
},
{
"id": "social_bio_profile",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram bio ideas", "LinkedIn profile tips", "Twitter bio generator"],
"pattern": "(.+?)\\s+(?:bio|profile)\\s+(?:ideas?|tips?|generator|examples?)",
"template": {
"like": "${1} bio ideas",
"where": { "domain": "social", "type": "profile" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "social_username",
"category": "domain_specific",
"domain": "social",
"examples": ["username ideas aesthetic", "check username availability", "Instagram username generator"],
"pattern": "(?:username|handle)\\s+(.+)",
"template": {
"like": "username ${1}",
"where": { "domain": "social", "type": "username" }
},
"confidence": 0.89,
"frequency": "medium"
},
{
"id": "social_caption",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram caption ideas", "funny captions", "caption for selfie"],
"pattern": "(.+?)\\s*(?:captions?|quotes?)\\s+(?:for\\s+)?(.+)?",
"template": {
"like": "${1} caption ${2}",
"where": { "domain": "social", "type": "caption" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "social_story_reel",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram story ideas", "how to make reels", "TikTok vs Reels"],
"pattern": "(?:Instagram\\s+)?(?:story|stories|reels?)\\s+(.+)",
"template": {
"like": "story reels ${1}",
"where": { "domain": "social", "type": "stories" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "social_monetization",
"category": "domain_specific",
"domain": "social",
"examples": ["monetize Instagram", "YouTube earnings calculator", "TikTok creator fund"],
"pattern": "(?:monetize|earn\\s+money|creator\\s+fund)\\s+(?:on\\s+)?(.+)",
"template": {
"like": "monetize ${1}",
"where": { "domain": "social", "type": "monetization" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "social_verification",
"category": "domain_specific",
"domain": "social",
"examples": ["get verified on Instagram", "Twitter blue checkmark", "verification requirements"],
"pattern": "(?:get\\s+)?verifi(?:ed|cation)\\s+(?:on\\s+)?(.+)",
"template": {
"like": "${1} verification",
"where": { "domain": "social", "type": "verification" }
},
"confidence": 0.92,
"frequency": "medium"
},
{
"id": "social_collaboration",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram collaboration", "brand partnerships", "influencer marketing"],
"pattern": "(?:brand\\s+)?(?:collaboration|partnership|sponsorship)\\s+(.+)",
"template": {
"like": "${1} collaboration",
"where": { "domain": "social", "type": "collaboration" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "social_dm_messaging",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram DM not working", "Twitter DM limits", "LinkedIn message templates"],
"pattern": "(.+?)\\s+(?:DM|direct\\s+message|messaging)\\s*(.+)?",
"template": {
"like": "${1} messaging ${2}",
"where": { "domain": "social", "type": "messaging" }
},
"confidence": 0.89,
"frequency": "medium"
},
{
"id": "social_privacy_settings",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram privacy settings", "make Twitter private", "Facebook privacy"],
"pattern": "(.+?)\\s+privacy\\s*(?:settings?)?",
"template": {
"like": "${1} privacy",
"where": { "domain": "social", "type": "privacy" }
},
"confidence": 0.91,
"frequency": "medium"
},
{
"id": "social_live_streaming",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram live tips", "YouTube streaming setup", "Twitch vs YouTube"],
"pattern": "(.+?)\\s+(?:live|streaming|stream)\\s*(.+)?",
"template": {
"like": "${1} live streaming ${2}",
"where": { "domain": "social", "type": "streaming" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "social_filters_effects",
"category": "domain_specific",
"domain": "social",
"examples": ["Instagram filters", "TikTok effects", "Snapchat lenses"],
"pattern": "(.+?)\\s+(?:filters?|effects?|lenses?)\\s*(.+)?",
"template": {
"like": "${1} filters ${2}",
"where": { "domain": "social", "type": "filters" }
},
"confidence": 0.88,
"frequency": "medium"
},
{
"id": "social_meme_viral",
"category": "domain_specific",
"domain": "social",
"examples": ["trending memes", "meme generator", "viral video ideas"],
"pattern": "(?:trending\\s+)?(?:memes?|viral\\s+videos?)\\s*(.+)?",
"template": {
"like": "memes viral ${1}",
"where": { "domain": "social", "type": "meme" }
},
"confidence": 0.89,
"frequency": "high"
}
]
}

487
src/patterns/tech-programming-patterns.json
View file

@ -1,487 +0,0 @@
{
"version": "2.0.0",
"description": "Programming, AI, and Tech domain patterns - HIGH PRIORITY",
"patterns": [
{
"id": "prog_error_message",
"category": "domain_specific",
"domain": "programming",
"examples": ["TypeError cannot read property", "undefined is not a function", "NullPointerException Java"],
"pattern": "([A-Za-z]+Error|[A-Za-z]+Exception)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "programming", "type": "error" }
},
"confidence": 0.96,
"frequency": "very_high"
},
{
"id": "prog_how_to_code",
"category": "domain_specific",
"domain": "programming",
"examples": ["how to reverse string Python", "sort array JavaScript", "read file in Java"],
"pattern": "(?:how\\s+to\\s+)?(.+?)\\s+(?:in|using)\\s+(Python|JavaScript|Java|C\\+\\+|TypeScript|Go|Rust|Ruby|PHP|Swift|Kotlin|C#)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "programming", "language": "${2}" }
},
"confidence": 0.95,
"frequency": "very_high"
},
{
"id": "prog_install_package",
"category": "domain_specific",
"domain": "programming",
"examples": ["npm install react", "pip install tensorflow", "cargo add tokio"],
"pattern": "(?:npm|pip|yarn|cargo|gem|composer|go get|brew|apt|yum)\\s+(?:install|add|get)\\s+(.+)",
"template": {
"like": "install ${1}",
"where": { "domain": "programming", "type": "package_install" }
},
"confidence": 0.94,
"frequency": "very_high"
},
{
"id": "prog_import_module",
"category": "domain_specific",
"domain": "programming",
"examples": ["import React from react", "from sklearn import", "require module Node.js"],
"pattern": "(?:import|from|require|use|include)\\s+(.+?)\\s+(?:from|in)?\\s*(.+)?",
"template": {
"like": "import ${1} ${2}",
"where": { "domain": "programming", "type": "import" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "prog_debug_issue",
"category": "domain_specific",
"domain": "programming",
"examples": ["debug React hooks", "memory leak Java", "segmentation fault C++"],
"pattern": "(?:debug|fix|solve|troubleshoot)\\s+(.+?)\\s*(?:issue|problem|error|bug)?",
"template": {
"like": "debug ${1}",
"where": { "domain": "programming", "type": "debugging" }
},
"confidence": 0.93,
"frequency": "very_high"
},
{
"id": "prog_best_practices",
"category": "domain_specific",
"domain": "programming",
"examples": ["React best practices", "Python coding standards", "clean code JavaScript"],
"pattern": "(.+?)\\s+(?:best\\s+practices?|coding\\s+standards?|clean\\s+code|style\\s+guide)",
"template": {
"like": "${1} best practices",
"where": { "domain": "programming", "type": "best_practices" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "prog_framework_tutorial",
"category": "domain_specific",
"domain": "programming",
"examples": ["React tutorial", "Django getting started", "Spring Boot guide"],
"pattern": "(React|Vue|Angular|Django|Flask|Spring|Express|Rails|Laravel|Next\\.js|Nuxt|FastAPI)\\s+(?:tutorial|guide|getting\\s+started)",
"template": {
"like": "${1} tutorial",
"where": { "domain": "programming", "framework": "${1}" }
},
"confidence": 0.94,
"frequency": "very_high"
},
{
"id": "prog_convert_code",
"category": "domain_specific",
"domain": "programming",
"examples": ["convert Python to JavaScript", "JSON to XML", "SQL to MongoDB"],
"pattern": "(?:convert|translate|transform)\\s+(.+?)\\s+to\\s+(.+)",
"template": {
"like": "convert ${1} to ${2}",
"where": { "domain": "programming", "type": "conversion" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "prog_api_docs",
"category": "domain_specific",
"domain": "programming",
"examples": ["OpenAI API documentation", "Stripe API reference", "REST API example"],
"pattern": "(.+?)\\s+API\\s+(?:documentation|reference|example|tutorial)",
"template": {
"like": "${1} API documentation",
"where": { "domain": "programming", "type": "api" }
},
"confidence": 0.93,
"frequency": "very_high"
},
{
"id": "prog_git_commands",
"category": "domain_specific",
"domain": "programming",
"examples": ["git merge conflict", "git rebase vs merge", "git undo commit"],
"pattern": "git\\s+(.+)",
"template": {
"like": "git ${1}",
"where": { "domain": "programming", "type": "version_control" }
},
"confidence": 0.95,
"frequency": "very_high"
},
{
"id": "prog_regex_pattern",
"category": "domain_specific",
"domain": "programming",
"examples": ["regex email validation", "regular expression phone number", "regex match URL"],
"pattern": "(?:regex|regular\\s+expression)\\s+(?:for\\s+)?(.+)",
"template": {
"like": "regex ${1}",
"where": { "domain": "programming", "type": "regex" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "prog_algorithm",
"category": "domain_specific",
"domain": "programming",
"examples": ["quicksort algorithm", "binary search implementation", "Dijkstra's algorithm"],
"pattern": "(.+?)\\s+(?:algorithm|implementation)\\s*(?:in\\s+(.+))?",
"template": {
"like": "${1} algorithm ${2}",
"where": { "domain": "programming", "type": "algorithm" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "prog_data_structure",
"category": "domain_specific",
"domain": "programming",
"examples": ["linked list vs array", "implement stack Python", "binary tree traversal"],
"pattern": "(?:implement\\s+)?(.+?)\\s*(?:data\\s+structure|vs\\.?\\s+(.+))?\\s*(?:in\\s+(.+))?",
"template": {
"like": "${1} data structure ${2} ${3}",
"where": { "domain": "programming", "type": "data_structure" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "ai_model_training",
"category": "domain_specific",
"domain": "ai",
"examples": ["train BERT model", "fine-tune GPT", "train neural network"],
"pattern": "(?:train|fine-?tune)\\s+(.+?)\\s*(?:model|network)?",
"template": {
"like": "train ${1}",
"where": { "domain": "ai", "type": "training" }
},
"confidence": 0.93,
"frequency": "very_high"
},
{
"id": "ai_machine_learning",
"category": "domain_specific",
"domain": "ai",
"examples": ["random forest sklearn", "neural network PyTorch", "CNN TensorFlow"],
"pattern": "(random\\s+forest|neural\\s+network|CNN|RNN|LSTM|transformer|SVM|k-?means)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "ai", "type": "ml_algorithm" }
},
"confidence": 0.92,
"frequency": "very_high"
},
{
"id": "ai_dataset",
"category": "domain_specific",
"domain": "ai",
"examples": ["MNIST dataset", "ImageNet download", "COCO dataset"],
"pattern": "(MNIST|ImageNet|COCO|CIFAR|WikiText|GLUE|SQuAD)\\s*(?:dataset)?\\s*(.+)?",
"template": {
"like": "${1} dataset ${2}",
"where": { "domain": "ai", "type": "dataset" }
},
"confidence": 0.94,
"frequency": "high"
},
{
"id": "ai_metrics",
"category": "domain_specific",
"domain": "ai",
"examples": ["accuracy vs precision", "F1 score calculation", "ROC curve explained"],
"pattern": "(accuracy|precision|recall|F1\\s+score|ROC|AUC|loss|perplexity)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "ai", "type": "metrics" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "ai_framework_comparison",
"category": "domain_specific",
"domain": "ai",
"examples": ["TensorFlow vs PyTorch", "Keras or TensorFlow", "JAX vs PyTorch"],
"pattern": "(TensorFlow|PyTorch|Keras|JAX|MXNet|Caffe|Theano)\\s+(?:vs\\.?|or)\\s+(.+)",
"template": {
"like": "${1} vs ${2}",
"where": { "domain": "ai", "type": "framework_comparison" }
},
"confidence": 0.93,
"frequency": "very_high"
},
{
"id": "ai_pretrained_model",
"category": "domain_specific",
"domain": "ai",
"examples": ["BERT pretrained model", "download GPT-2", "use ResNet50"],
"pattern": "(BERT|GPT|GPT-2|GPT-3|GPT-4|ResNet|VGG|YOLO|EfficientNet)\\s+(?:pretrained\\s+)?(?:model)?\\s*(.+)?",
"template": {
"like": "${1} pretrained ${2}",
"where": { "domain": "ai", "type": "pretrained_model" }
},
"confidence": 0.94,
"frequency": "very_high"
},
{
"id": "ai_nlp_task",
"category": "domain_specific",
"domain": "ai",
"examples": ["sentiment analysis Python", "named entity recognition", "text classification BERT"],
"pattern": "(sentiment\\s+analysis|NER|named\\s+entity|text\\s+classification|summarization|translation)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "ai", "type": "nlp" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "ai_computer_vision",
"category": "domain_specific",
"domain": "ai",
"examples": ["object detection YOLO", "image segmentation", "face recognition OpenCV"],
"pattern": "(object\\s+detection|image\\s+segmentation|face\\s+recognition|OCR|image\\s+classification)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "ai", "type": "computer_vision" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "tech_cloud_service",
"category": "domain_specific",
"domain": "tech",
"examples": ["AWS S3 tutorial", "Google Cloud pricing", "Azure vs AWS"],
"pattern": "(AWS|Azure|GCP|Google\\s+Cloud|Heroku|DigitalOcean|Vercel|Netlify)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "cloud" }
},
"confidence": 0.93,
"frequency": "very_high"
},
{
"id": "tech_docker_kubernetes",
"category": "domain_specific",
"domain": "tech",
"examples": ["Docker compose example", "Kubernetes deployment", "dockerfile for Node.js"],
"pattern": "(Docker|Kubernetes|K8s|container|dockerfile|docker-compose)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "containerization" }
},
"confidence": 0.92,
"frequency": "very_high"
},
{
"id": "tech_database_query",
"category": "domain_specific",
"domain": "tech",
"examples": ["SQL join example", "MongoDB aggregation", "PostgreSQL vs MySQL"],
"pattern": "(SQL|MySQL|PostgreSQL|MongoDB|Redis|Elasticsearch|Cassandra)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "database" }
},
"confidence": 0.94,
"frequency": "very_high"
},
{
"id": "tech_devops_ci_cd",
"category": "domain_specific",
"domain": "tech",
"examples": ["GitHub Actions workflow", "Jenkins pipeline", "CI/CD best practices"],
"pattern": "(GitHub\\s+Actions|Jenkins|CircleCI|Travis|GitLab\\s+CI|CI/CD)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "devops" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "tech_security",
"category": "domain_specific",
"domain": "tech",
"examples": ["SQL injection prevention", "XSS attack", "JWT authentication"],
"pattern": "(SQL\\s+injection|XSS|CSRF|JWT|OAuth|authentication|authorization)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "security" }
},
"confidence": 0.93,
"frequency": "high"
},
{
"id": "tech_performance",
"category": "domain_specific",
"domain": "tech",
"examples": ["optimize React performance", "database indexing", "lazy loading implementation"],
"pattern": "(?:optimize|improve)\\s+(.+?)\\s+performance",
"template": {
"like": "${1} performance optimization",
"where": { "domain": "tech", "type": "performance" }
},
"confidence": 0.92,
"frequency": "high"
},
{
"id": "tech_testing",
"category": "domain_specific",
"domain": "tech",
"examples": ["unit testing Jest", "integration testing", "mock API calls"],
"pattern": "(unit\\s+test|integration\\s+test|e2e\\s+test|mock|stub)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "testing" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "prog_stackoverflow_pattern",
"category": "domain_specific",
"domain": "programming",
"examples": ["undefined is not a function JavaScript", "cannot read property of undefined React"],
"pattern": "(.+?)\\s+(JavaScript|Python|Java|C\\+\\+|React|Angular|Vue)$",
"template": {
"like": "${1} ${2}",
"where": { "domain": "programming", "source": "stackoverflow" }
},
"confidence": 0.95,
"frequency": "very_high"
},
{
"id": "prog_vscode_extension",
"category": "domain_specific",
"domain": "programming",
"examples": ["VSCode extension Python", "best VSCode themes", "VSCode shortcuts"],
"pattern": "(?:VSCode|VS\\s+Code)\\s+(.+)",
"template": {
"like": "VSCode ${1}",
"where": { "domain": "programming", "type": "ide" }
},
"confidence": 0.90,
"frequency": "medium"
},
{
"id": "ai_llm_models",
"category": "domain_specific",
"domain": "ai",
"examples": ["ChatGPT API", "Claude vs GPT-4", "Llama 2 fine-tuning"],
"pattern": "(ChatGPT|Claude|GPT-4|Llama|Mistral|Gemini|DALL-E|Stable\\s+Diffusion)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "ai", "type": "llm" }
},
"confidence": 0.95,
"frequency": "very_high"
},
{
"id": "ai_prompt_engineering",
"category": "domain_specific",
"domain": "ai",
"examples": ["prompt engineering tips", "ChatGPT prompts", "system prompt examples"],
"pattern": "(?:prompt\\s+engineering|prompts?|system\\s+prompt)\\s+(.+)",
"template": {
"like": "prompt engineering ${1}",
"where": { "domain": "ai", "type": "prompt_engineering" }
},
"confidence": 0.93,
"frequency": "very_high"
},
{
"id": "tech_web_framework",
"category": "domain_specific",
"domain": "tech",
"examples": ["Next.js vs Gatsby", "Tailwind CSS tutorial", "Bootstrap components"],
"pattern": "(Next\\.js|Gatsby|Tailwind|Bootstrap|Material-UI|Chakra|Ant\\s+Design)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "web_framework" }
},
"confidence": 0.92,
"frequency": "very_high"
},
{
"id": "tech_mobile_dev",
"category": "domain_specific",
"domain": "tech",
"examples": ["React Native navigation", "Flutter vs React Native", "SwiftUI tutorial"],
"pattern": "(React\\s+Native|Flutter|SwiftUI|Kotlin|Swift|Android|iOS)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "mobile" }
},
"confidence": 0.91,
"frequency": "high"
},
{
"id": "prog_package_version",
"category": "domain_specific",
"domain": "programming",
"examples": ["React 18 features", "Python 3.11 new", "Node.js version 20"],
"pattern": "(.+?)\\s+(?:version\\s+)?(\\d+(?:\\.\\d+)*)\\s*(.+)?",
"template": {
"like": "${1} ${2} ${3}",
"where": { "domain": "programming", "version": "${2}" }
},
"confidence": 0.90,
"frequency": "high"
},
{
"id": "tech_cli_commands",
"category": "domain_specific",
"domain": "tech",
"examples": ["curl POST request", "wget download file", "ssh key generation"],
"pattern": "(curl|wget|ssh|scp|rsync|grep|sed|awk|chmod|chown)\\s+(.+)",
"template": {
"like": "${1} command ${2}",
"where": { "domain": "tech", "type": "cli" }
},
"confidence": 0.92,
"frequency": "very_high"
},
{
"id": "tech_linux_admin",
"category": "domain_specific",
"domain": "tech",
"examples": ["Ubuntu install package", "systemd service", "cron job example"],
"pattern": "(Ubuntu|Debian|CentOS|Linux|systemd|cron|iptables|nginx|apache)\\s+(.+)",
"template": {
"like": "${1} ${2}",
"where": { "domain": "tech", "type": "sysadmin" }
},
"confidence": 0.91,
"frequency": "high"
}
]
}

3
src/plugin.ts
View file

@ -5,7 +5,8 @@
* 1. Native acceleration (@soulcraft/cortex)
* 2. Custom storage adapters (e.g., Redis, DynamoDB, custom backends)
*
* Plugins are auto-detected by package name or registered manually.
* Plugins are loaded from an explicit `plugins: [...]` config list or
* registered manually via `brain.use()` — there is no implicit detection.
*/
import type {

127
src/scripts/precomputePatternEmbeddings.ts
View file

@ -1,127 +0,0 @@
#!/usr/bin/env node
/**
* 🧠 Pre-compute Pattern Embeddings Script
*
* This script pre-computes embeddings for all patterns and saves them to disk.
* Run this once after adding new patterns to avoid runtime embedding costs.
*
* How it works:
* 1. Load all patterns from library.json
* 2. Use Brainy's embedding model to encode each pattern's examples
* 3. Average the example embeddings to get a robust pattern representation
* 4. Save embeddings to patterns/embeddings.bin for instant loading
*
* Benefits:
* - Pattern matching becomes pure math (cosine similarity)
* - No embedding model calls during query processing
* - Patterns load instantly with pre-computed vectors
*/
import { Brainy } from '../brainy.js'
import patternData from '../patterns/library.json' assert { type: 'json' }
import * as fs from 'node:fs/promises'
import * as path from 'node:path'
async function precomputeEmbeddings() {
console.log('🧠 Pre-computing pattern embeddings...')
// Initialize Brainy with minimal config
const brain = new Brainy({
storage: 'memory',
verbose: false
})
await brain.init()
console.log('✅ Brainy initialized')
const embeddings: Record<string, {
patternId: string
embedding: number[]
examples: string[]
averageMethod: string
}> = {}
let processedCount = 0
const totalPatterns = patternData.patterns.length
for (const pattern of patternData.patterns) {
console.log(`\n📝 Processing pattern: ${pattern.id} (${++processedCount}/${totalPatterns})`)
console.log(` Category: ${pattern.category}`)
console.log(` Examples: ${pattern.examples.length}`)
// Embed all examples
const exampleEmbeddings: number[][] = []
for (const example of pattern.examples) {
try {
const embedding = await brain.embed(example)
exampleEmbeddings.push(embedding as number[])
console.log(` ✓ Embedded: "${example.substring(0, 50)}..."`)
} catch (error) {
console.error(` ✗ Failed to embed: "${example}"`, error)
}
}
if (exampleEmbeddings.length === 0) {
console.warn(` ⚠️ No embeddings generated for pattern ${pattern.id}`)
continue
}
// Average the embeddings for a robust representation
const avgEmbedding = averageVectors(exampleEmbeddings)
embeddings[pattern.id] = {
patternId: pattern.id,
embedding: avgEmbedding,
examples: pattern.examples,
averageMethod: 'arithmetic_mean'
}
console.log(` ✅ Generated ${avgEmbedding.length}-dimensional embedding`)
}
// Save embeddings to file
const outputPath = path.join(process.cwd(), 'src', 'patterns', 'embeddings.json')
await fs.writeFile(outputPath, JSON.stringify(embeddings, null, 2))
console.log(`\n✅ Saved ${Object.keys(embeddings).length} pattern embeddings to ${outputPath}`)
// Calculate storage size
const stats = await fs.stat(outputPath)
console.log(`📊 File size: ${(stats.size / 1024).toFixed(2)} KB`)
// Print statistics
console.log('\n📈 Embedding Statistics:')
console.log(` Total patterns: ${totalPatterns}`)
console.log(` Successfully embedded: ${Object.keys(embeddings).length}`)
console.log(` Failed: ${totalPatterns - Object.keys(embeddings).length}`)
console.log(` Embedding dimensions: ${Object.values(embeddings)[0]?.embedding.length || 0}`)
await brain.close()
console.log('\n✅ Complete!')
}
function averageVectors(vectors: number[][]): number[] {
if (vectors.length === 0) return []
const dim = vectors[0].length
const avg = new Array(dim).fill(0)
// Sum all vectors
for (const vec of vectors) {
for (let i = 0; i < dim; i++) {
avg[i] += vec[i]
}
}
// Divide by count to get average
for (let i = 0; i < dim; i++) {
avg[i] /= vectors.length
}
return avg
}
// Run the script
precomputeEmbeddings().catch(console.error)

8
src/storage/adapters/fileSystemStorage.ts
View file

@ -1612,6 +1612,8 @@ export class FileSystemStorage extends BaseStorage {
} else {
const stale = await this.isWriterLockStale(existing)
if (!stale) {
// Consumer-facing error contract: callers detect this case via
// err.code and read the holder's details from err.lockInfo.
const err = new Error(
`Another writer holds this Brainy directory.\n` +
` PID: ${existing.pid} on host ${existing.hostname}\n` +
@ -1622,9 +1624,9 @@ export class FileSystemStorage extends BaseStorage {
`For diagnostic queries against this live store, use:\n` +
` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', rootDirectory: '${this.rootDir}' } })\n\n` +
`If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.`
)
;(err as any).code = 'BRAINY_WRITER_LOCKED'
;(err as any).lockInfo = existing
) as Error & { code: string; lockInfo: WriterLockInfo }
err.code = 'BRAINY_WRITER_LOCKED'
err.lockInfo = existing
throw err
}
console.warn(

119
src/storage/baseStorage.ts
View file

@ -193,6 +193,27 @@ function getVerbMetadataPath(id: string): string {
return `entities/verbs/${shard}/${id}/metadata.json`
}
/**
* Optional count capabilities probed via duck typing by getNouns()/getVerbs().
* Adapters with a native O(1) count API may implement these; they are not part
* of the BaseStorageAdapter contract, so BaseStorage feature-detects them at
* runtime before falling back to scan-based counting.
*/
interface OptionalCountCapabilities {
countNouns?: (filter?: {
nounType?: string | string[]
service?: string | string[]
metadata?: Record<string, unknown>
}) => Promise<number>
countVerbs?: (filter?: {
verbType?: string | string[]
sourceId?: string | string[]
targetId?: string | string[]
service?: string | string[]
metadata?: Record<string, unknown>
}) => Promise<number>
}
/**
* Base storage adapter that implements common functionality
* This is an abstract class that should be extended by specific storage adapters
@ -454,9 +475,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public invalidateGraphIndex(): void {
if (this.graphIndex) {
prodLog.info('[BaseStorage] Invalidating GraphAdjacencyIndex for clear()')
// Stop any pending operations
if (typeof (this.graphIndex as any).stopAutoFlush === 'function') {
(this.graphIndex as any).stopAutoFlush()
// Stop any pending operations. stopAutoFlush is an optional capability
// of plugin-provided graph indexes (duck-typed; the built-in
// GraphAdjacencyIndex does not implement it).
const flushable = this.graphIndex as GraphAdjacencyIndex & {
stopAutoFlush?: () => void
}
if (typeof flushable.stopAutoFlush === 'function') {
flushable.stopAutoFlush()
}
this.graphIndex = undefined
this.graphIndexPromise = undefined
@ -738,7 +764,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*
* The hook fires once per entity-visible metadata mutation — noun/verb
* metadata saves and deletes, the one storage write every logical Brainy
* mutation (`add`, `update`, `delete`, `relate`, `updateRelation`,
* mutation (`add`, `update`, `remove`, `relate`, `updateRelation`,
* `unrelate`) performs exactly once per entity it touches. It does NOT fire
* for derived-index writes (HNSW node data, metadata-index chunks, LSM
* segments, statistics), so the generation counter tracks *data* mutations,
@ -1380,9 +1406,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// First, try to get a count of total nouns (if the adapter supports it)
let totalCount: number | undefined = undefined
try {
// This is an optional method that adapters may implement
if (typeof (this as any).countNouns === 'function') {
totalCount = await (this as any).countNouns(options?.filter)
// This is an optional method that adapters may implement (duck-typed —
// see OptionalCountCapabilities)
const adapter = this as BaseStorage & OptionalCountCapabilities
if (typeof adapter.countNouns === 'function') {
totalCount = await adapter.countNouns(options?.filter)
}
} catch (countError) {
// Ignore errors from count method, it's optional
@ -1390,9 +1418,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
}
// Check if the adapter has a paginated method for getting nouns
if (typeof (this as any).getNounsWithPagination === 'function') {
// Use the adapter's paginated method - pass offset directly to adapter
const result = await (this as any).getNounsWithPagination({
if (typeof this.getNounsWithPagination === 'function') {
// Use the adapter's paginated method - pass offset directly to adapter.
// The annotation widens totalCount to optional: adapter overrides
// follow BaseStorageAdapter's contract, where totalCount may be absent.
const result: {
items: HNSWNounWithMetadata[]
totalCount?: number
hasMore: boolean
nextCursor?: string
} = await this.getNounsWithPagination({
limit,
offset, // Let the adapter handle offset for O(1) operation
cursor,
@ -1556,7 +1591,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
/**
* Get verbs with pagination (Type-first implementation with billion-scale optimizations)
*
* CRITICAL: This method is required for brain.getRelations() to work!
* CRITICAL: This method is required for brain.related() to work!
* Iterates through verb types with the same optimizations as nouns.
*
* ARCHITECTURE: Reads storage directly (not indexes) to avoid circular dependencies.
@ -1605,11 +1640,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const filterTargetIds = filter?.targetId
? new Set(Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId])
: null
const filterSubtypes = (filter as any)?.subtype
// `subtype` rides alongside the declared filter fields (Brainy's
// related() path passes it through); it's applied after metadata
// loads below, since subtype lives in verb metadata, not on the raw verb.
const subtypeFilterValue = (
filter as { verbType?: string | string[]; subtype?: string | string[] } | undefined
)?.subtype
const filterSubtypes = subtypeFilterValue
? new Set(
Array.isArray((filter as any).subtype)
? (filter as any).subtype
: [(filter as any).subtype]
Array.isArray(subtypeFilterValue)
? subtypeFilterValue
: [subtypeFilterValue]
)
: null
@ -1652,7 +1693,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Apply subtype filter (requires metadata — checked AFTER load)
if (filterSubtypes) {
const subtype = (metadata as any)?.subtype as string | undefined
const subtype = metadata?.subtype as string | undefined
if (!subtype || !filterSubtypes.has(subtype)) {
continue
}
@ -1725,9 +1766,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// subtype check after the load). Subtype is not stored on the raw HNSWVerb;
// it's a metadata field. The graph-index fast paths return verbs without
// metadata, so they can't apply subtype filtering correctly.
if (options?.filter && !(options.filter as any).subtype) {
if (
options?.filter &&
!(options.filter as { verbType?: string | string[]; subtype?: string | string[] }).subtype
) {
// CRITICAL VFS FIX: If filtering by sourceId + verbType (most common VFS pattern!)
// This is the query PathResolver.getChildren() uses: getRelations({ from: dirId, type: VerbType.Contains })
// This is the query PathResolver.getChildren() uses: related({ from: dirId, type: VerbType.Contains })
if (
options.filter.sourceId &&
options.filter.verbType &&
@ -1921,9 +1965,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// First, try to get a count of total verbs (if the adapter supports it)
let totalCount: number | undefined = undefined
try {
// This is an optional method that adapters may implement
if (typeof (this as any).countVerbs === 'function') {
totalCount = await (this as any).countVerbs(options?.filter)
// This is an optional method that adapters may implement (duck-typed —
// see OptionalCountCapabilities)
const adapter = this as BaseStorage & OptionalCountCapabilities
if (typeof adapter.countVerbs === 'function') {
totalCount = await adapter.countVerbs(options?.filter)
}
} catch (countError) {
// Ignore errors from count method, it's optional
@ -1931,13 +1977,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
}
// Check if the adapter has a paginated method for getting verbs
if (typeof (this as any).getVerbsWithPagination === 'function') {
if (typeof this.getVerbsWithPagination === 'function') {
// Use the adapter's paginated method
// Convert offset to cursor if no cursor provided (adapters use cursor for offset)
const effectiveCursor = cursor || (offset > 0 ? offset.toString() : undefined)
const result = await (this as any).getVerbsWithPagination({
// offset is carried via effectiveCursor for adapters with cursor-based
// pagination; 0 here matches the long-standing destructure default in
// getVerbsWithPagination (this call never passed offset). The
// annotation widens totalCount to optional: adapter overrides follow
// BaseStorageAdapter's contract, where totalCount may be absent.
const result: {
items: HNSWVerbWithMetadata[]
totalCount?: number
hasMore: boolean
nextCursor?: string
} = await this.getVerbsWithPagination({
limit,
offset: 0,
cursor: effectiveCursor,
filter: options?.filter
})
@ -2398,7 +2455,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
* **Use cases:**
* - VFS tree traversal (fetch all children at once)
* - brain.find() result hydration (batch load entities)
* - brain.getRelations() target entities (eliminate N+1)
* - brain.related() target entities (eliminate N+1)
* - Import operations (batch existence checks)
*
* @param ids Array of entity IDs to fetch
@ -2571,8 +2628,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
private async readBatchFromAdapter(paths: string[]): Promise<Map<string, any>> {
if (paths.length === 0) return new Map()
// Check if this class implements batch operations (will be added to cloud adapters)
const selfWithBatch = this as any
// Check if this class implements batch operations (will be added to cloud
// adapters). Duck-typed optional capability — readBatch is not part of the
// BaseStorageAdapter contract.
const selfWithBatch = this as BaseStorage & {
readBatch?: (paths: string[]) => Promise<Map<string, unknown>>
}
if (typeof selfWithBatch.readBatch === 'function') {
// Adapter has native batch support - use it
@ -2693,7 +2754,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
await this.ensureInitialized()
// Extract verb type from metadata for ID-first path
const verbType = (metadata as any).verb as VerbType | undefined
const verbType = metadata.verb as VerbType | undefined
if (!verbType) {
// Backward compatibility: fallback to old path if no verb type
@ -2729,7 +2790,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype)
} else if (!priorEntry && priorVerbForSubtype && !isNew) {
// Edge case: cache miss but metadata existed with a subtype (e.g. reader process startup).
const priorSubFromMeta = (existingMetadata as any)?.subtype as string | undefined
const priorSubFromMeta = existingMetadata?.subtype as string | undefined
if (priorSubFromMeta && (priorSubFromMeta !== newSubtype || priorVerbForSubtype !== verbType)) {
this.decrementVerbSubtypeCount(priorVerbForSubtype, priorSubFromMeta)
}
@ -3706,7 +3767,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
*
* **Use cases:**
* - VFS tree traversal (get Contains edges for multiple directories)
* - brain.getRelations() for multiple entities
* - brain.related() for multiple entities
* - Graph traversal (fetch neighbors of multiple nodes)
*
* @param sourceIds Array of source entity IDs

66
src/streaming/pipeline.ts
View file

@ -55,7 +55,20 @@ export class Pipeline<T = any> {
}
constructor(private brainyInstance?: Brainy | Brainy<any>) {}
/**
* Re-brand this pipeline's element type after a stage that changes the
* stream's item type has been pushed. The runtime object is unchanged —
* the fluent builder mutates `stages` on the same instance rather than
* allocating a new pipeline, and `Pipeline<T>`'s type parameter exists
* only at compile time. (Typed boundary: `Pipeline<T>` and `Pipeline<R>`
* are not structurally comparable generics, so this is the one sanctioned
* `unknown` bridge in this class.)
*/
private retype<R>(): Pipeline<R> {
return this as unknown as Pipeline<R>
}
/**
* Add a data source
*/
@ -71,9 +84,9 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<S>()
}
/**
* Transform data
*/
@ -88,7 +101,7 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<R>()
}
/**
@ -143,7 +156,12 @@ export class Pipeline<T = any> {
if (result) {
// Note: This won't work perfectly in async iterator
// In production, use a proper queue
batch = [result as any]
// Typed boundary preserving a pre-existing quirk verbatim:
// re-stashing the flushed batch as a single element nests it
// (a T[] stored where T is expected), so a later size-flush
// can yield a nested array. Documented, not fixed — fixing it
// (batch = result) would change runtime output.
batch = [result] as unknown as T[]
}
}, timeoutMs)
}
@ -155,9 +173,9 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<T[]>()
}
/**
* Sink data to a destination
*/
@ -172,7 +190,7 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<void>()
}
/**
@ -197,7 +215,8 @@ export class Pipeline<T = any> {
for (const item of batch) {
await (brain as Brainy<any>).add({
data: item,
type: (options?.type as any) || NounType.Document,
// Type coercion since pipeline accepts string
type: (options?.type as NounType | undefined) || NounType.Document,
metadata: options?.metadata
})
}
@ -206,12 +225,12 @@ export class Pipeline<T = any> {
for (const item of batch) {
await (brain as Brainy).add({
data: item,
type: (options?.type || 'document') as any, // Type coercion since pipeline accepts string
type: (options?.type || 'document') as NounType, // Type coercion since pipeline accepts string
metadata: options?.metadata
})
}
}
}) as any
})
}
/**
@ -244,7 +263,7 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<void>()
}
/**
@ -282,7 +301,7 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<void>()
}
/**
@ -332,7 +351,7 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<T[]>()
}
/**
@ -352,7 +371,7 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<R>()
}
/**
@ -411,7 +430,7 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<R>()
}
/**
@ -522,7 +541,7 @@ export class Pipeline<T = any> {
}
}
this.stages.push(stage)
return this as any
return this.retype<R>()
}
/**
@ -542,12 +561,17 @@ export class Pipeline<T = any> {
const { errorHandler, bufferSize = 1000 } = options
try {
// Build the pipeline chain
let stream: AsyncIterable<any> = undefined as any
// Build the pipeline chain. The head of the chain has no upstream:
// source stages declare process() without parameters and never read
// their input, so the undefined fed to them (and to a malformed
// pipeline whose first stage isn't a source — preserved legacy
// behavior) is typed at this boundary as the AsyncIterable it never is.
const noUpstream = undefined as unknown as AsyncIterable<unknown>
let stream: AsyncIterable<unknown> = noUpstream
for (const stage of this.stages) {
if (stage.type === 'source') {
stream = stage.process(undefined as any)
stream = stage.process(noUpstream)
} else {
stream = stage.process(stream)
}

21
src/transaction/operations/IndexOperations.ts
View file

@ -50,13 +50,26 @@ export class AddToHNSWOperation implements Operation {
}
/**
* Check if item exists in index
* Check if item exists in index.
*
* `getItem` is an optional, feature-detected provider capability — see the
* VectorIndexProvider docs; it is intentionally absent from the required
* contract (Brainy's JS HNSW index omits it). When the capability is
* missing the answer must be `false`, not `true`: treating unknowable
* pre-existence as "existed" made every rollback skip removeItem, leaving
* phantom entries in the index after a failed transaction. The safe default
* is to remove what this operation added — update flows pair this op with a
* RemoveFromHNSWOperation whose own rollback restores the prior vector, so
* reverse-order rollback reconstructs the original state either way.
*/
private async itemExists(id: string): Promise<boolean> {
const index = this.index as JsHnswVectorIndex & {
getItem?: (id: string) => Promise<unknown>
}
if (typeof index.getItem !== 'function') return false
try {
// Try to get item - if exists, no error
await (this.index as any).getItem?.(id)
return true
const item = await index.getItem(id)
return item !== undefined && item !== null
} catch {
return false
}

8
src/transaction/operations/StorageOperations.ts
View file

@ -95,8 +95,8 @@ export class SaveNounOperation implements Operation {
// Delete newly created noun (if adapter supports it)
// Note: Not all adapters implement deleteNoun
// This is acceptable - metadata deletion makes entity invisible
if ('deleteNoun' in this.storage && typeof (this.storage as any).deleteNoun === 'function') {
await (this.storage as any).deleteNoun(this.noun.id)
if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') {
await this.storage.deleteNoun(this.noun.id)
}
}
}
@ -211,8 +211,8 @@ export class SaveVerbOperation implements Operation {
await this.storage.saveVerb(verbData)
} else {
// Delete newly created verb (if adapter supports it)
if ('deleteVerb' in this.storage && typeof (this.storage as any).deleteVerb === 'function') {
await (this.storage as any).deleteVerb(this.verb.id)
if ('deleteVerb' in this.storage && typeof this.storage.deleteVerb === 'function') {
await this.storage.deleteVerb(this.verb.id)
}
}
}

2
src/types/apiTypes.ts
View file

@ -188,7 +188,7 @@ export interface ConfigUpdateParams {
/**
* API for Triple Intelligence Engine to access Brainy internals
* This provides type-safe access without 'as any' casts
* This provides type-safe access without unchecked casts
*/
export interface TripleIntelligenceAPI {
// Vector operations

44
src/types/brainy.types.ts
View file

@ -86,7 +86,7 @@ export interface Relation<T = any> {
* relationship might have `subtype: 'direct'` vs `'dotted-line'`; a `RelatedTo`
* edge might carry `'spouse'` / `'sibling'` / `'colleague'`). Flat string, no
* hierarchy. Top-level standard field — indexed on the fast path and rolled into
* per-VerbType statistics so queries like `getRelations({ verb, subtype })` hit
* per-VerbType statistics so queries like `related({ verb, subtype })` hit
* the column-store directly.
*/
subtype?: string
@ -372,7 +372,7 @@ export interface RelateParams<T = any> {
* relationship might have `subtype: 'direct'` or `'dotted-line'`; a `RelatedTo`
* edge might carry `'spouse'` or `'colleague'`). Flat string, no hierarchy.
* Indexed and rolled up into per-VerbType statistics for fast filtering
* (`getRelations({ verb, subtype })`) and aggregation (`groupBy:['subtype']`).
* (`related({ verb, subtype })`) and aggregation (`groupBy:['subtype']`).
*/
subtype?: string
/** Connection strength (0-1, default: 1.0) */
@ -542,7 +542,7 @@ export interface SimilarParams<T = any> {
}
/**
* Parameters for getting relationships
* Parameters for `brain.related()` / `db.related()`
*
* All parameters are optional. When called without parameters, returns all relationships
* with pagination (default limit: 100).
@ -550,25 +550,25 @@ export interface SimilarParams<T = any> {
* @example
* ```typescript
* // Get all relationships (default limit: 100)
* const all = await brain.getRelations()
* const all = await brain.related()
*
* // Get relationships from a specific entity (string shorthand)
* const fromEntity = await brain.getRelations(entityId)
* const fromEntity = await brain.related(entityId)
*
* // Equivalent to:
* const fromEntity2 = await brain.getRelations({ from: entityId })
* const fromEntity2 = await brain.related({ from: entityId })
*
* // Get relationships to a specific entity
* const toEntity = await brain.getRelations({ to: entityId })
* const toEntity = await brain.related({ to: entityId })
*
* // Filter by relationship type
* const friends = await brain.getRelations({ type: VerbType.FriendOf })
* const friends = await brain.related({ type: VerbType.FriendOf })
*
* // Pagination
* const page2 = await brain.getRelations({ offset: 100, limit: 50 })
* const page2 = await brain.related({ offset: 100, limit: 50 })
*
* // Combined filters
* const filtered = await brain.getRelations({
* const filtered = await brain.related({
* from: entityId,
* type: VerbType.WorksWith,
* limit: 20
@ -578,7 +578,7 @@ export interface SimilarParams<T = any> {
* Fixed bug where calling without parameters returned empty array
* Added string ID shorthand syntax
*/
export interface GetRelationsParams {
export interface RelatedParams {
/**
* Filter by source entity ID
*
@ -668,15 +668,15 @@ export interface UpdateManyParams<T = any> {
}
/**
* Batch delete parameters
* Batch remove parameters
*/
export interface DeleteManyParams {
ids?: string[] // Specific IDs to delete
type?: NounType // Delete all of type
where?: any // Delete by metadata
limit?: number // Max to delete (safety)
export interface RemoveManyParams {
ids?: string[] // Specific IDs to remove
type?: NounType // Remove all of type
where?: any // Remove by metadata
limit?: number // Max to remove (safety)
onProgress?: (done: number, total: number) => void
continueOnError?: boolean // Continue processing if a delete fails
continueOnError?: boolean // Continue processing if a removal fails
}
/**
@ -854,7 +854,7 @@ export interface ImportResult {
* - VFS operations (readFile, stat, readdir) - 100% of cases
* - Existence checks: `if (await brain.get(id))`
* - Metadata inspection: `entity.metadata`, `entity.data`, `entity.type`
* - Relationship traversal: `brain.getRelations({ from: id })`
* - Relationship traversal: `brain.related({ from: id })`
* - Search operations: `brain.find()` generates embeddings automatically
*
* @example
@ -1364,16 +1364,14 @@ export interface BrainyConfig {
*
* - `'writer'` (default): acquires an exclusive lock on the storage directory at
* `init()` and refuses to open if another live writer holds the lock. Required
* for any instance that calls `add`/`update`/`delete`/`relate` or any other
* for any instance that calls `add`/`update`/`remove`/`relate` or any other
* mutation. Released on `close()` and on process exit/SIGINT/SIGTERM.
* - `'reader'`: opens without acquiring the writer lock. Coexists with a live
* writer and with other readers. All mutation methods throw
* `Cannot mutate a read-only Brainy instance`. Use `Brainy.openReadOnly()`
* as a convenience factory.
*
* For cloud storage backends (s3/r2/gcs/azure) the lock is a best-effort
* advisory marker — multi-process safety against concurrent writers on
* cloud-backed stores is not yet enforced. See `docs/concepts/multi-process.md`.
* See `docs/concepts/multi-process.md` for the full coordination model.
*/
mode?: 'writer' | 'reader'

2
src/types/progress.types.ts
View file

@ -3,8 +3,6 @@
*
* Provides unified progress tracking across all long-running operations
* in Brainy (imports, clustering, large searches, etc.)
*
* PRODUCTION-READY - NO MOCKS, NO STUBS, REAL IMPLEMENTATION
*/
/**

12
src/unified.ts
View file

@ -18,8 +18,18 @@ export const environment = {
}
}
declare global {
/**
* Runtime-environment marker published by the unified entry point so
* embedders and diagnostics can detect the active Brainy environment.
* Ambient `var` is required — `let`/`const` in `declare global` do not
* attach to `globalThis`.
*/
var __ENV__: typeof environment | undefined
}
if (typeof globalThis !== 'undefined') {
;(globalThis as any).__ENV__ = environment
globalThis.__ENV__ = environment
}
console.log(

119
src/universal/events.ts
View file

@ -1,119 +0,0 @@
/**
* Universal Events implementation
* Framework-friendly: Trusts that frameworks provide events polyfills
* Works in all environments: Browser (via framework), Node.js, Serverless
*/
import { isNode } from '../utils/environment.js'
let nodeEvents: any = null
// Dynamic import for Node.js events (only in Node.js environment)
if (isNode()) {
try {
nodeEvents = await import('node:events')
} catch {
// Ignore import errors in non-Node environments
}
}
/**
* Universal EventEmitter interface
*/
export interface UniversalEventEmitter {
on(event: string, listener: (...args: any[]) => void): this
off(event: string, listener: (...args: any[]) => void): this
emit(event: string, ...args: any[]): boolean
once(event: string, listener: (...args: any[]) => void): this
removeAllListeners(event?: string): this
listenerCount(event: string): number
}
/**
* Node.js implementation using events.EventEmitter
*/
class NodeEventEmitter implements UniversalEventEmitter {
private emitter: any
constructor() {
this.emitter = new nodeEvents.EventEmitter()
}
on(event: string, listener: (...args: any[]) => void): this {
this.emitter.on(event, listener)
return this
}
off(event: string, listener: (...args: any[]) => void): this {
this.emitter.off(event, listener)
return this
}
emit(event: string, ...args: any[]): boolean {
return this.emitter.emit(event, ...args)
}
once(event: string, listener: (...args: any[]) => void): this {
this.emitter.once(event, listener)
return this
}
removeAllListeners(event?: string): this {
this.emitter.removeAllListeners(event)
return this
}
listenerCount(event: string): number {
return this.emitter.listenerCount(event)
}
}
/**
* Universal EventEmitter class
* Framework-friendly: Assumes events API is available via framework polyfills
*/
export class EventEmitter implements UniversalEventEmitter {
private emitter: UniversalEventEmitter
constructor() {
if (isNode() && nodeEvents) {
this.emitter = new NodeEventEmitter()
} else {
throw new Error('Events operations not available. Framework bundlers should provide events polyfills.')
}
}
on(event: string, listener: (...args: any[]) => void): this {
this.emitter.on(event, listener)
return this
}
off(event: string, listener: (...args: any[]) => void): this {
this.emitter.off(event, listener)
return this
}
emit(event: string, ...args: any[]): boolean {
return this.emitter.emit(event, ...args)
}
once(event: string, listener: (...args: any[]) => void): this {
this.emitter.once(event, listener)
return this
}
removeAllListeners(event?: string): this {
this.emitter.removeAllListeners(event)
return this
}
listenerCount(event: string): number {
return this.emitter.listenerCount(event)
}
}
// Named export for compatibility
export { EventEmitter as default }
// Re-export Node.js EventEmitter class if available
export const NodeEventEmitterClass = nodeEvents?.EventEmitter || null

28
src/universal/index.ts
View file

@ -1,28 +0,0 @@
/**
* Universal adapters for cross-environment compatibility
* Provides consistent APIs across Browser, Node.js, and Serverless environments
*/
// UUID adapter
export * from './uuid.js'
export { default as uuid } from './uuid.js'
// Crypto adapter
export * from './crypto.js'
export { default as crypto } from './crypto.js'
// File system adapter
export * from './fs.js'
export { default as fs } from './fs.js'
// Path adapter
export * from './path.js'
export { default as path } from './path.js'
// Events adapter
export * from './events.js'
export { default as events } from './events.js'
// Convenience re-exports for common patterns
export { v4 as uuidv4 } from './uuid.js'
export { EventEmitter } from './events.js'

2
src/utils/callerLocation.ts
View file

@ -38,7 +38,7 @@ export function findCallerLocation(extraSkipPatterns: string[] = []): string | n
for (const raw of lines) {
const line = raw.trim()
// Always skip Brainy's own source + compiled output. Consumers need their
// own call site, not `brainy.ts:XXXX` or `dist/brainy.js:XXXX`.
// own call site, not a line inside `brainy.ts` or `dist/brainy.js`.
if (line.includes('/src/brainy.ts') || line.includes('/dist/brainy.js')) continue
// Skip the validation + diagnostic helpers regardless of which file they
// live in — they're plumbing between the public API and the consumer.

2
src/utils/embedding.ts
View file

@ -120,7 +120,7 @@ export function createTransformerEmbedding(options: TransformerEmbeddingOptions
* Convenience function to detect best device (always returns 'wasm')
*/
export async function detectBestDevice(): Promise<'cpu' | 'webgpu' | 'cuda' | 'wasm'> {
return 'wasm' as any
return 'wasm'
}
/**

296
src/utils/enhancedLogger.ts
View file

@ -1,296 +0,0 @@
/**
* Enhanced logging system that bridges old logger with new structured logger
* Provides backward compatibility while enabling gradual migration
*/
import {
structuredLogger,
createModuleLogger as createStructuredModuleLogger,
LogLevel as StructuredLogLevel,
type LogContext,
type ModuleLogger
} from './structuredLogger.js'
import { isProductionEnvironment, getLogLevel } from './environment.js'
// Re-export LogLevel for compatibility
export enum LogLevel {
ERROR = 0,
WARN = 1,
INFO = 2,
DEBUG = 3,
TRACE = 4
}
// Map old LogLevel to new StructuredLogLevel
function mapLogLevel(level: LogLevel): StructuredLogLevel {
switch (level) {
case LogLevel.ERROR: return StructuredLogLevel.ERROR
case LogLevel.WARN: return StructuredLogLevel.WARN
case LogLevel.INFO: return StructuredLogLevel.INFO
case LogLevel.DEBUG: return StructuredLogLevel.DEBUG
case LogLevel.TRACE: return StructuredLogLevel.TRACE
default: return StructuredLogLevel.INFO
}
}
export interface LoggerConfig {
level: LogLevel
modules?: {
[moduleName: string]: LogLevel
}
timestamps?: boolean
includeModule?: boolean
handler?: (level: LogLevel, module: string, message: string, ...args: any[]) => void
}
/**
* Enhanced Logger that uses structured logger internally
* Maintains backward compatibility with existing code
*/
class EnhancedLogger {
private static instance: EnhancedLogger
private config: LoggerConfig = {
level: LogLevel.ERROR,
timestamps: false,
includeModule: true
}
private constructor() {
this.applyEnvironmentDefaults()
// Sync with structured logger
this.syncWithStructuredLogger()
// Set log level from environment variable if available
const envLogLevel = process.env.BRAINY_LOG_LEVEL
if (envLogLevel) {
const level = LogLevel[envLogLevel.toUpperCase() as keyof typeof LogLevel]
if (level !== undefined) {
this.config.level = level
this.syncWithStructuredLogger()
}
}
// Parse module-specific log levels
const moduleLogLevels = process.env.BRAINY_MODULE_LOG_LEVELS
if (moduleLogLevels) {
try {
this.config.modules = JSON.parse(moduleLogLevels)
this.syncWithStructuredLogger()
} catch (e) {
// Ignore parsing errors
}
}
}
private applyEnvironmentDefaults(): void {
const envLogLevel = getLogLevel()
switch (envLogLevel) {
case 'silent':
this.config.level = -1 as LogLevel
break
case 'error':
this.config.level = LogLevel.ERROR
this.config.timestamps = false
break
case 'warn':
this.config.level = LogLevel.WARN
this.config.timestamps = false
break
case 'info':
this.config.level = LogLevel.INFO
this.config.timestamps = true
break
case 'verbose':
this.config.level = LogLevel.DEBUG
this.config.timestamps = true
break
}
if (isProductionEnvironment()) {
this.config.level = Math.min(this.config.level, LogLevel.ERROR)
this.config.timestamps = false
this.config.includeModule = false
}
}
private syncWithStructuredLogger(): void {
// Convert modules config
const modules: Record<string, StructuredLogLevel> = {}
if (this.config.modules) {
for (const [module, level] of Object.entries(this.config.modules)) {
modules[module] = mapLogLevel(level)
}
}
// Configure structured logger
structuredLogger.configure({
level: mapLogLevel(this.config.level),
modules,
format: this.config.timestamps ? 'pretty' : 'simple'
})
}
static getInstance(): EnhancedLogger {
if (!EnhancedLogger.instance) {
EnhancedLogger.instance = new EnhancedLogger()
}
return EnhancedLogger.instance
}
configure(config: Partial<LoggerConfig>): void {
this.config = { ...this.config, ...config }
this.syncWithStructuredLogger()
}
private shouldLog(level: LogLevel, module: string): boolean {
if (this.config.modules && this.config.modules[module] !== undefined) {
return level <= this.config.modules[module]
}
return level <= this.config.level
}
error(module: string, message: string, ...args: any[]): void {
const logger = createStructuredModuleLogger(module)
logger.error(message, { data: args })
}
warn(module: string, message: string, ...args: any[]): void {
const logger = createStructuredModuleLogger(module)
logger.warn(message, { data: args })
}
info(module: string, message: string, ...args: any[]): void {
const logger = createStructuredModuleLogger(module)
logger.info(message, { data: args })
}
debug(module: string, message: string, ...args: any[]): void {
const logger = createStructuredModuleLogger(module)
logger.debug(message, { data: args })
}
trace(module: string, message: string, ...args: any[]): void {
const logger = createStructuredModuleLogger(module)
logger.trace(message, { data: args })
}
createModuleLogger(module: string) {
const structuredLogger = createStructuredModuleLogger(module)
// Return a backward-compatible interface
return {
error: (message: string, ...args: any[]) =>
structuredLogger.error(message, { data: args }),
warn: (message: string, ...args: any[]) =>
structuredLogger.warn(message, { data: args }),
info: (message: string, ...args: any[]) =>
structuredLogger.info(message, { data: args }),
debug: (message: string, ...args: any[]) =>
structuredLogger.debug(message, { data: args }),
trace: (message: string, ...args: any[]) =>
structuredLogger.trace(message, { data: args }),
// New structured logging methods
withContext: (context: LogContext) =>
structuredLogger.withContext(context),
startTimer: (label: string) =>
structuredLogger.startTimer(label),
endTimer: (label: string) =>
structuredLogger.endTimer(label)
}
}
}
// Export singleton instance
export const logger = EnhancedLogger.getInstance()
// Export convenience function for creating module loggers
export function createModuleLogger(module: string) {
return logger.createModuleLogger(module)
}
// Export function to configure logger
export function configureLogger(config: Partial<LoggerConfig>) {
logger.configure(config)
}
/**
* Smart console replacement that uses structured logger
*/
export const smartConsole = {
log: (message?: any, ...args: any[]) => {
const logger = createStructuredModuleLogger('console')
logger.info(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
},
info: (message?: any, ...args: any[]) => {
const logger = createStructuredModuleLogger('console')
logger.info(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
},
warn: (message?: any, ...args: any[]) => {
const logger = createStructuredModuleLogger('console')
logger.warn(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
},
error: (message?: any, ...args: any[]) => {
const logger = createStructuredModuleLogger('console')
logger.error(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
},
debug: (message?: any, ...args: any[]) => {
const logger = createStructuredModuleLogger('console')
logger.debug(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
},
trace: (message?: any, ...args: any[]) => {
const logger = createStructuredModuleLogger('console')
logger.trace(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
}
}
/**
* Production-optimized logging functions
*/
export const prodLog = {
error: (message?: any, ...args: any[]) => {
const logger = createStructuredModuleLogger('prod')
logger.error(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
},
warn: (message?: any, ...args: any[]) => {
if (!isProductionEnvironment() || process.env.BRAINY_LOG_LEVEL) {
const logger = createStructuredModuleLogger('prod')
logger.warn(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
}
},
info: (message?: any, ...args: any[]) => {
if (!isProductionEnvironment() || process.env.BRAINY_LOG_LEVEL) {
const logger = createStructuredModuleLogger('prod')
logger.info(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
}
},
debug: (message?: any, ...args: any[]) => {
if (!isProductionEnvironment() || process.env.BRAINY_LOG_LEVEL) {
const logger = createStructuredModuleLogger('prod')
logger.debug(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
}
},
log: (message?: any, ...args: any[]) => {
if (!isProductionEnvironment() || process.env.BRAINY_LOG_LEVEL) {
const logger = createStructuredModuleLogger('prod')
logger.info(typeof message === 'string' ? message : JSON.stringify(message), { data: args })
}
}
}
// Re-export structured logger utilities for new code
export {
createModuleLogger as createStructuredModuleLogger,
type LogContext,
type ModuleLogger
} from './structuredLogger.js'

9
src/utils/entityIdMapper.ts
View file

@ -124,8 +124,11 @@ export class EntityIdMapper implements EntityIdMapperProvider {
try {
const metadata = await this.storage.getMetadata(this.storageKey)
// metadata IS the data (no nested 'data' property)
if (metadata && (metadata as any).nextId !== undefined) {
const data = metadata as any as EntityIdMapperData
if (metadata && metadata.nextId !== undefined) {
// Typed boundary: mapper state round-trips through the storage
// metadata channel as plain JSON; the `nextId` probe above identifies
// the persisted EntityIdMapperData shape.
const data = metadata as unknown as EntityIdMapperData
this.nextId = data.nextId
// Rebuild maps from serialized data
@ -305,7 +308,7 @@ export class EntityIdMapper implements EntityIdMapperProvider {
intToUuid: Object.fromEntries(this.intToUuid)
}
await this.storage.saveMetadata(this.storageKey, data as any)
await this.storage.saveMetadata(this.storageKey, data)
this.dirty = false
}

10
src/utils/fieldTypeInference.ts
View file

@ -22,7 +22,7 @@
* 4. Store result for future queries
*/
import { StorageAdapter } from '../coreTypes.js'
import { StorageAdapter, NounMetadata } from '../coreTypes.js'
import { prodLog } from './logger.js'
/**
@ -412,7 +412,7 @@ export class FieldTypeInference {
noun: 'FieldTypeCache',
...typeInfo
}
await this.storage.saveMetadata(cacheKey, metadataObj as any).catch(error => {
await this.storage.saveMetadata(cacheKey, metadataObj).catch(error => {
prodLog.warn(`Failed to save field type cache for '${field}':`, error)
})
}
@ -487,8 +487,10 @@ export class FieldTypeInference {
if (field) {
this.typeCache.delete(field)
const cacheKey = `${this.CACHE_STORAGE_PREFIX}${field}`
// null signals deletion to storage adapter
await this.storage.saveMetadata(cacheKey, null as any)
// Typed boundary: the storage metadata channel doubles as the delete
// path — writing a JSON `null` tombstone clears the cached entry, but
// the adapter signature only models real payloads.
await this.storage.saveMetadata(cacheKey, null as unknown as NounMetadata)
} else {
this.typeCache.clear()
}

393
src/utils/intelligentTypeMapper.ts
View file

@ -1,393 +0,0 @@
/**
* Intelligent Type Mapper
* Maps generic/invalid type names to specific semantic types based on data analysis
* Prevents semantic degradation from overuse of generic types
*/
import { NounType, VerbType } from '../types/graphTypes.js'
/**
* Common aliases that users might use
*/
const GENERIC_ALIASES = new Set([
'entity',
'item',
'object',
'node',
'record',
'entry',
'data',
'resource'
])
/**
* Field signatures for type inference
*/
const TYPE_SIGNATURES = {
// Person indicators
person: {
required: [],
indicators: ['email', 'firstName', 'lastName', 'name', 'phone', 'username', 'userId'],
patterns: [/@/, /^[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}$/i],
weight: 10
},
// User account indicators
user: {
required: [],
indicators: ['username', 'password', 'accountId', 'loginTime', 'permissions', 'role'],
patterns: [],
weight: 9
},
// Organization indicators
organization: {
required: [],
indicators: ['companyName', 'orgName', 'ein', 'vatNumber', 'employees', 'headquarters'],
patterns: [],
weight: 8
},
// Product indicators
product: {
required: [],
indicators: ['price', 'sku', 'barcode', 'inventory', 'cost', 'productId', 'inStock'],
patterns: [/^\$?\d+\.?\d*$/, /^[A-Z0-9-]+$/],
weight: 8
},
// Document indicators
document: {
required: [],
indicators: ['content', 'text', 'body', 'title', 'author', 'markdown', 'html'],
patterns: [],
weight: 7
},
// Message indicators
message: {
required: [],
indicators: ['from', 'to', 'subject', 'body', 'sentAt', 'messageId', 'threadId'],
patterns: [],
weight: 7
},
// Task indicators
task: {
required: [],
indicators: ['dueDate', 'assignee', 'status', 'priority', 'completed', 'taskId'],
patterns: [],
weight: 7
},
// Event indicators
event: {
required: [],
indicators: ['startTime', 'endTime', 'date', 'location', 'attendees', 'eventType'],
patterns: [],
weight: 6
},
// Location indicators
location: {
required: [],
indicators: ['latitude', 'longitude', 'address', 'city', 'country', 'zipCode', 'coordinates'],
patterns: [/^-?\d+\.\d+$/, /^\d{5}(-\d{4})?$/],
weight: 6
},
// File indicators
file: {
required: [],
indicators: ['filename', 'filepath', 'extension', 'mimeType', 'fileSize', 'checksum'],
patterns: [/\.[a-z0-9]+$/i],
weight: 5
},
// Dataset indicators
dataset: {
required: [],
indicators: ['schema', 'rows', 'columns', 'records', 'dataType', 'format'],
patterns: [],
weight: 5
},
// Media indicators
media: {
required: [],
indicators: ['url', 'thumbnail', 'duration', 'resolution', 'codec', 'bitrate'],
patterns: [/\.(jpg|jpeg|png|gif|mp4|mp3|wav|avi)$/i],
weight: 5
},
// Project indicators
project: {
required: [],
indicators: ['deadline', 'budget', 'team', 'milestones', 'deliverables', 'projectId'],
patterns: [],
weight: 5
},
// Service indicators
service: {
required: [],
indicators: ['endpoint', 'apiKey', 'serviceUrl', 'port', 'protocol', 'healthCheck'],
patterns: [/^https?:\/\//, /:\d+$/],
weight: 4
}
}
/**
* Intelligent Type Mapper
*/
export class IntelligentTypeMapper {
private typeCache: Map<string, string> = new Map()
private inferenceStats = {
total: 0,
inferred: 0,
defaulted: 0,
cached: 0
}
/**
* Map a noun type, with intelligent inference for generic types
*/
mapNounType(inputType: string, data?: any): string {
// Check if it's already a valid type
if (this.isValidNounType(inputType)) {
return inputType
}
// Check cache for this exact input
const cacheKey = `${inputType}-${JSON.stringify(data || {}).substring(0, 100)}`
if (this.typeCache.has(cacheKey)) {
this.inferenceStats.cached++
return this.typeCache.get(cacheKey)!
}
this.inferenceStats.total++
// If it's a generic alias and we have data, try to infer
if (GENERIC_ALIASES.has(inputType.toLowerCase()) && data) {
const inferred = this.inferTypeFromData(data)
if (inferred) {
this.inferenceStats.inferred++
this.typeCache.set(cacheKey, inferred)
return inferred
}
}
// Handle specific common mappings
const directMapping = this.getDirectMapping(inputType)
if (directMapping) {
this.typeCache.set(cacheKey, directMapping)
return directMapping
}
// Default to 'thing' for truly unknown types
this.inferenceStats.defaulted++
const defaultType = NounType.Thing
this.typeCache.set(cacheKey, defaultType)
return defaultType
}
/**
* Map a verb type
*/
mapVerbType(inputType: string): string {
// Check if it's already valid
if (this.isValidVerbType(inputType)) {
return inputType
}
// Common verb mappings
const verbMappings: Record<string, string> = {
'related': VerbType.RelatedTo,
'relates': VerbType.RelatedTo,
'has': VerbType.Contains,
'includes': VerbType.Contains,
'belongsTo': VerbType.PartOf,
'in': VerbType.LocatedAt,
'at': VerbType.LocatedAt,
'references': VerbType.References,
'cites': VerbType.References,
'before': VerbType.Precedes,
'after': VerbType.Precedes,
'causes': VerbType.Causes,
'needs': VerbType.Requires,
'requires': VerbType.Requires,
'makes': VerbType.Creates,
'produces': VerbType.Creates,
'changes': VerbType.Modifies,
'updates': VerbType.Modifies,
'owns': VerbType.Owns,
'ownedBy': VerbType.Owns, // Use BelongsTo for reverse ownership
'uses': VerbType.Uses,
'usedBy': VerbType.Uses // Same relationship, just interpret direction
}
const normalized = inputType.toLowerCase()
return verbMappings[normalized] || VerbType.RelatedTo
}
/**
* Infer type from data structure
*/
private inferTypeFromData(data: any): string | null {
if (!data || typeof data !== 'object') return null
const scores: Map<string, number> = new Map()
const fields = Object.keys(data)
const values = Object.values(data)
// Calculate scores for each type based on field matches
for (const [type, signature] of Object.entries(TYPE_SIGNATURES)) {
let score = 0
// Check required fields
if (signature.required.length > 0) {
const hasRequired = signature.required.every(field => fields.includes(field))
if (!hasRequired) continue
score += signature.weight * 2
}
// Check indicator fields
for (const field of fields) {
if (signature.indicators.some(indicator =>
field.toLowerCase().includes(indicator.toLowerCase())
)) {
score += signature.weight
}
}
// Check value patterns
for (const value of values) {
if (typeof value === 'string' && signature.patterns.length > 0) {
for (const pattern of signature.patterns) {
if (pattern.test(value)) {
score += signature.weight / 2
break
}
}
}
}
if (score > 0) {
scores.set(type, score)
}
}
// Return the type with highest score
if (scores.size > 0) {
const sorted = Array.from(scores.entries()).sort((a, b) => b[1] - a[1])
return sorted[0][0]
}
// Fallback inference based on data structure
if (fields.includes('url') || fields.includes('href')) {
return NounType.Document
}
if (Array.isArray(data) || fields.includes('items') || fields.includes('elements')) {
return NounType.Collection
}
// Check if it looks like a process/workflow
if (fields.includes('steps') || fields.includes('stages')) {
return NounType.Process
}
return null
}
/**
* Get direct mapping for common aliases
*/
private getDirectMapping(inputType: string): string | null {
const mappings: Record<string, string> = {
// Specific mappings that aren't generic
'company': NounType.Organization,
'corp': NounType.Organization,
'business': NounType.Organization,
'employee': NounType.Person,
'staff': NounType.Person,
'customer': NounType.Person,
'client': NounType.Person,
'article': NounType.Document,
'post': NounType.Document,
'page': NounType.Document,
'image': NounType.Media,
'video': NounType.Media,
'audio': NounType.Media,
'photo': NounType.Media,
'place': NounType.Location,
'address': NounType.Location,
'country': NounType.Location,
'city': NounType.Location,
'todo': NounType.Task,
'job': NounType.Task,
'work': NounType.Task,
'meeting': NounType.Event,
'appointment': NounType.Event,
'conference': NounType.Event,
'folder': NounType.Collection,
'group': NounType.Collection,
'list': NounType.Collection,
'category': NounType.Collection
}
const normalized = inputType.toLowerCase()
return mappings[normalized] || null
}
/**
* Check if a type is valid
*/
private isValidNounType(type: string): boolean {
return Object.values(NounType).includes(type as any)
}
/**
* Check if a verb type is valid
*/
private isValidVerbType(type: string): boolean {
return Object.values(VerbType).includes(type as any)
}
/**
* Get inference statistics
*/
getStats() {
return {
...this.inferenceStats,
cacheSize: this.typeCache.size,
inferenceRate: this.inferenceStats.total > 0
? (this.inferenceStats.inferred / this.inferenceStats.total)
: 0
}
}
/**
* Clear the type cache
*/
clearCache(): void {
this.typeCache.clear()
}
}
// Singleton instance
export const typeMapper = new IntelligentTypeMapper()
/**
* Helper function for easy type mapping
*/
export function mapNounType(inputType: string, data?: any): string {
return typeMapper.mapNounType(inputType, data)
}
/**
* Helper function for verb mapping
*/
export function mapVerbType(inputType: string): string {
return typeMapper.mapVerbType(inputType)
}

34
src/utils/metadataIndex.ts
View file

@ -4,7 +4,7 @@
* Automatically updates indexes when data changes
*/
import { StorageAdapter, resolveEntityField } from '../coreTypes.js'
import { StorageAdapter, resolveEntityField, NounMetadata, VerbMetadata } from '../coreTypes.js'
import { ColumnStore } from '../indexes/columnStore/ColumnStore.js'
import type { MetadataIndexProvider } from '../plugin.js'
import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
@ -1520,13 +1520,13 @@ export class MetadataIndexManager implements MetadataIndexProvider {
const allIds = new Set<string>()
// Storage.getNouns() is the definitive source of all entity IDs
if (this.storage && typeof (this.storage as any).getNouns === 'function') {
if (this.storage && typeof this.storage.getNouns === 'function') {
try {
const result = await (this.storage as any).getNouns({
const result = await this.storage.getNouns({
pagination: { limit: 100000 }
})
if (result && result.items) {
result.items.forEach((item: any) => {
result.items.forEach((item) => {
if (item.id) allIds.add(item.id)
})
}
@ -2446,7 +2446,7 @@ export class MetadataIndexManager implements MetadataIndexProvider {
noun: 'MetadataFieldIndex',
values: fieldIndex.values,
lastUpdated: fieldIndex.lastUpdated
} as any)
})
// Update unified cache
const size = JSON.stringify(fieldIndex).length
@ -2582,8 +2582,11 @@ export class MetadataIndexManager implements MetadataIndexProvider {
await this.chunkManager.deleteChunk(field, chunkId)
}
// Delete the sparse index file itself
await this.storage.saveMetadata(indexPath, null as any)
// Delete the sparse index file itself.
// Typed boundary: the storage metadata channel doubles as the delete
// path — writing a JSON `null` tombstone clears the entry, but the
// adapter signature only models real payloads.
await this.storage.saveMetadata(indexPath, null as unknown as NounMetadata)
}
} catch (error) {
// Silent failure - if we can't delete old chunks, rebuild will still work
@ -2612,9 +2615,11 @@ export class MetadataIndexManager implements MetadataIndexProvider {
deletedCount++
}
// Delete field registry
// Delete field registry.
// Typed boundary: writing a JSON `null` tombstone clears the entry, but
// the adapter signature only models real payloads.
try {
await this.storage.saveMetadata('__metadata_field_registry__', null as any)
await this.storage.saveMetadata('__metadata_field_registry__', null as unknown as NounMetadata)
} catch (error) {
prodLog.debug('Could not delete field registry:', error)
}
@ -3104,10 +3109,15 @@ export class MetadataIndexManager implements MetadataIndexProvider {
prodLog.info(`📦 Loading ${result.items.length} verbs with metadata...`)
const verbIds = result.items.map(verb => verb.id)
let verbMetadataBatch: Map<string, any>
let verbMetadataBatch: Map<string, VerbMetadata>
if ((this.storage as any).getVerbMetadataBatch) {
verbMetadataBatch = await (this.storage as any).getVerbMetadataBatch(verbIds)
// Optional adapter capability: batched verb-metadata reads. Not part of
// the StorageAdapter contract, so it is probed structurally.
const batchCapableStorage = this.storage as StorageAdapter & {
getVerbMetadataBatch?: (ids: string[]) => Promise<Map<string, VerbMetadata>>
}
if (batchCapableStorage.getVerbMetadataBatch) {
verbMetadataBatch = await batchCapableStorage.getVerbMetadataBatch(verbIds)
prodLog.info(`✅ Loaded ${verbMetadataBatch.size}/${verbIds.length} verb metadata objects`)
} else {
verbMetadataBatch = new Map()

50
src/utils/metadataIndexChunking.ts
View file

@ -20,7 +20,7 @@
* - EntityIdMapper handles UUID ↔ integer conversion
*/
import { StorageAdapter } from '../coreTypes.js'
import { StorageAdapter, NounMetadata } from '../coreTypes.js'
import { prodLog } from './logger.js'
import { RoaringBitmap32 } from './roaring/index.js'
import type { EntityIdMapper } from './entityIdMapper.js'
@ -109,6 +109,29 @@ export interface ChunkData {
lastUpdated: number
}
/**
* Storage representation of a roaring bitmap inside a serialized chunk.
* Produced by `ChunkManager.saveChunk()` — portable-format bytes as a plain
* number array so the payload survives JSON round-trips.
*/
type SerializedRoaringBitmap = {
buffer: number[]
size: number
}
/**
* Storage representation of a chunk as persisted by `ChunkManager.saveChunk()`
* and re-hydrated by `ChunkManager.loadChunk()`. A type alias (not an
* interface) so it carries an implicit index signature and converts cleanly
* to/from the `NounMetadata` shape used by the storage metadata channel.
*/
type SerializedChunkData = {
chunkId: number
field: string
entries: Record<string, SerializedRoaringBitmap | undefined>
lastUpdated: number
}
// ============================================================================
// BloomFilter - Production-Ready Implementation
// ============================================================================
@ -609,25 +632,26 @@ export class ChunkManager {
const data = await this.storage.getMetadata(chunkPath)
if (data) {
// Cast NounMetadata to chunk data structure
const chunkData = data as unknown as any
// Chunks round-trip through the storage metadata channel; re-type the
// JSON payload to the serialized chunk shape written by saveChunk()
const chunkData = data as SerializedChunkData
// Deserialize: convert serialized roaring bitmaps back to RoaringBitmap32 objects
const chunk: ChunkData = {
chunkId: chunkData.chunkId as number,
field: chunkData.field as string,
chunkId: chunkData.chunkId,
field: chunkData.field,
entries: new Map(
Object.entries(chunkData.entries).map(([value, serializedBitmap]) => {
// Deserialize roaring bitmap from portable format
const bitmap = new RoaringBitmap32()
if (serializedBitmap && typeof serializedBitmap === 'object' && (serializedBitmap as any).buffer) {
if (serializedBitmap && typeof serializedBitmap === 'object' && serializedBitmap.buffer) {
// Deserialize from Buffer
bitmap.deserialize(Buffer.from((serializedBitmap as any).buffer), 'portable')
bitmap.deserialize(Buffer.from(serializedBitmap.buffer), 'portable')
}
return [value, bitmap]
return [value, bitmap] as const
})
),
lastUpdated: chunkData.lastUpdated as number
lastUpdated: chunkData.lastUpdated
}
this.chunkCache.set(cacheKey, chunk)
@ -668,7 +692,7 @@ export class ChunkManager {
}
const chunkPath = this.getChunkPath(chunk.field, chunk.chunkId)
await this.storage.saveMetadata(chunkPath, serializable as any)
await this.storage.saveMetadata(chunkPath, serializable)
}
/**
@ -842,8 +866,10 @@ export class ChunkManager {
this.chunkCache.delete(cacheKey)
const chunkPath = this.getChunkPath(field, chunkId)
// null signals deletion to storage adapter
await this.storage.saveMetadata(chunkPath, null as any)
// Typed boundary: the storage metadata channel doubles as the delete path —
// writing a JSON `null` tombstone clears the chunk. The adapter signature
// only models real payloads, so the null must be re-typed here.
await this.storage.saveMetadata(chunkPath, null as unknown as NounMetadata)
}
/**

Some files were not shown because too many files have changed in this diff Show more