soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: CRITICAL systemic VFS metadata bug across ALL storage adapters (v4.7.4)

Browse source 
CRITICAL BUG FIX - Workshop Team Unblocked!

This hotfix resolves a systemic bug affecting ALL 7 storage adapters that
caused VFS queries to return empty results even when data existed.

Bug Pattern: `if (!metadata) continue` in getNouns()/getVerbs()
Impact: VFS queries returned empty arrays despite 577 relationships existing
Root Cause: Storage adapters skipped entities if metadata file read returned null

Fixes:
- storage: Fix metadata skip bug in 12 locations across 7 adapters
  (TypeAware, Memory, FileSystem, GCS, S3, R2, OPFS, Azure)
- neural: Fix SmartExtractor weighted score threshold (28 failures → 4)
- neural: Fix PatternSignal priority ordering
- api: Fix Brainy.relate() weight parameter not returned

Test Results:
- TypeAwareStorageAdapter: 17/17 passing (was 7 failures)
- SmartExtractor: 42/46 passing (was 28 failures)
- Neural clustering: 3/3 passing
- Brainy.relate(): 20/20 passing

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-10-27 14:23:46 -07:00
parent c75bbb9ba4
commit e06edb7d52
15 changed files with 287 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/brainy.ts
View file

@ -3026,7 +3026,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
from: v.sourceId,
to: v.targetId,
type: (v.verb || v.type) as VerbType,
weight: v.weight,
weight: v.metadata?.weight ?? 1.0, // v4.7.4: weight is in metadata
metadata: v.metadata,
service: v.metadata?.service as string,
createdAt: typeof v.createdAt === 'number' ? v.createdAt : Date.now()

17
src/neural/SmartExtractor.ts
View file

@ -556,8 +556,17 @@ export class SmartExtractor {
}
}
// Determine final confidence score
// FIX: When only one signal matches, use its original confidence instead of weighted score
// The weighted score is too low when only one signal matches (e.g., 0.8 * 0.2 = 0.16 < 0.60 threshold)
let finalConfidence = bestScore
if (bestSignals.length === 1) {
// Single signal: use its original confidence
finalConfidence = bestSignals[0].confidence
}
// Check minimum confidence threshold
if (!bestType || bestScore < this.options.minConfidence) {
if (!bestType || finalConfidence < this.options.minConfidence) {
return null
}
@ -572,7 +581,7 @@ export class SmartExtractor {
return {
type: bestType,
confidence: Math.min(bestScore, 1.0), // Cap at 1.0
confidence: Math.min(finalConfidence, 1.0), // Cap at 1.0
source: 'ensemble',
evidence,
metadata: {
@ -609,7 +618,9 @@ export class SmartExtractor {
const best = validResults[0]
if (best.weightedScore < this.options.minConfidence) {
// FIX: Use original confidence, not weighted score for threshold check
// Weighted score is for ranking signals, not for absolute threshold
if (best.confidence < this.options.minConfidence) {
return null
}

61
src/neural/signals/PatternSignal.ts
View file

@ -109,33 +109,58 @@ export class PatternSignal {
* - Document: files, papers, reports
*/
private initializePatterns(): void {
// Person patterns
this.addPatterns(NounType.Person, 0.80, [
/\b(?:Dr|Prof|Mr|Mrs|Ms|Sir|Lady|Lord)\s+[A-Z][a-z]+/,
/\b[A-Z][a-z]+\s+[A-Z][a-z]+(?:\s+[A-Z][a-z]+)?\b/, // Full names
/\b(?:CEO|CTO|CFO|COO|VP|Director|Manager|Engineer|Developer|Designer)\b/i,
/\b(?:author|creator|founder|inventor|contributor|maintainer)\b/i,
/\b(?:user|member|participant|attendee|speaker|presenter)\b/i
// Organization patterns - HIGH PRIORITY (must check before person full name pattern)
this.addPatterns(NounType.Organization, 0.88, [
/\b(?:Inc|LLC|Corp|Ltd|GmbH|SA|AG)\b/, // Strong org indicators
/\b[A-Z][a-z]+\s+(?:Company|Corporation|Enterprises|Industries|Group)\b/
])
// Location patterns
this.addPatterns(NounType.Location, 0.75, [
/\b(?:city|town|village|country|nation|state|province)\b/i,
/\b(?:street|avenue|road|boulevard|lane|drive)\b/i,
/\b(?:building|tower|center|complex|headquarters)\b/i,
/\b(?:north|south|east|west|central)\s+[A-Z][a-z]+/i,
/\b[A-Z][a-z]+,\s*[A-Z]{2}\b/ // City, State format
// Location patterns - HIGH PRIORITY (city/country format, addresses)
this.addPatterns(NounType.Location, 0.86, [
/\b[A-Z][a-z]+,\s*[A-Z]{2}\b/, // City, State format (e.g., "Paris, FR")
/\b(?:street|avenue|road|boulevard|lane|drive)\b/i
])
// Organization patterns
this.addPatterns(NounType.Organization, 0.78, [
/\b(?:Inc|LLC|Corp|Ltd|GmbH|SA|AG)\b/,
/\b[A-Z][a-z]+\s+(?:Company|Corporation|Enterprises|Industries|Group)\b/,
// Event patterns - HIGH PRIORITY (specific event keywords)
this.addPatterns(NounType.Event, 0.84, [
/\b(?:conference|summit|symposium|workshop|seminar|webinar)\b/i,
/\b(?:hackathon|bootcamp)\b/i
])
// Person patterns - SPECIFIC INDICATORS (high confidence)
this.addPatterns(NounType.Person, 0.82, [
/\b(?:Dr|Prof|Mr|Mrs|Ms|Sir|Lady|Lord)\s+[A-Z][a-z]+/, // Titles
/\b(?:CEO|CTO|CFO|COO|VP|Director|Manager|Engineer|Developer|Designer)\b/i, // Roles
/\b(?:author|creator|founder|inventor|contributor|maintainer)\b/i
])
// Organization patterns - MEDIUM PRIORITY
this.addPatterns(NounType.Organization, 0.76, [
/\b(?:university|college|institute|academy|school)\b/i,
/\b(?:department|division|team|committee|board)\b/i,
/\b(?:government|agency|bureau|ministry|administration)\b/i
])
// Location patterns - MEDIUM PRIORITY
this.addPatterns(NounType.Location, 0.74, [
/\b(?:city|town|village|country|nation|state|province)\b/i,
/\b(?:building|tower|center|complex|headquarters)\b/i,
/\b(?:north|south|east|west|central)\s+[A-Z][a-z]+/i
])
// Event patterns - MEDIUM PRIORITY
this.addPatterns(NounType.Event, 0.72, [
/\b(?:meeting|session|call|standup|retrospective|sprint)\b/i,
/\b(?:release|launch|deployment|rollout|update)\b/i,
/\b(?:training|course|tutorial)\b/i
])
// Person patterns - GENERIC (low confidence, catches full names but easily overridden)
this.addPatterns(NounType.Person, 0.68, [
/\b[A-Z][a-z]+\s+[A-Z][a-z]+(?:\s+[A-Z][a-z]+)?\b/, // Full names (generic, low priority)
/\b(?:user|member|participant|attendee|speaker|presenter)\b/i
])
// Technology patterns (Thing type)
this.addPatterns(NounType.Thing, 0.82, [
/\b(?:JavaScript|TypeScript|Python|Java|C\+\+|Go|Rust|Swift|Kotlin)\b/,

4
src/storage/adapters/azureBlobStorage.ts
View file

@ -1254,8 +1254,8 @@ export class AzureBlobStorage extends BaseStorage {
const node = await this.getNode(id)
if (!node) continue
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
const metadata = await this.getNounMetadata(id)
if (!metadata) continue
// Apply filters if provided
if (options.filter) {
@ -1274,7 +1274,7 @@ export class AzureBlobStorage extends BaseStorage {
// Combine node with metadata
items.push({
...node,
metadata
metadata: (metadata || {}) as NounMetadata // Empty if none
})
count++

6
src/storage/adapters/fileSystemStorage.ts
View file

@ -908,11 +908,11 @@ export class FileSystemStorage extends BaseStorage {
const parsedNoun = JSON.parse(data)
// v4.0.0: Load metadata from separate storage
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
const metadata = await this.getNounMetadata(id)
if (!metadata) continue
// Apply filter if provided
if (options.filter) {
if (options.filter && metadata) {
let matches = true
for (const [key, value] of Object.entries(options.filter)) {
if (metadata[key] !== value) {
@ -939,7 +939,7 @@ export class FileSystemStorage extends BaseStorage {
vector: parsedNoun.vector,
connections: connections,
level: parsedNoun.level || 0,
metadata: metadata
metadata: (metadata || {}) as NounMetadata // Empty if none
}
items.push(nounWithMetadata)

4
src/storage/adapters/gcsStorage.ts
View file

@ -1046,8 +1046,8 @@ export class GcsStorage extends BaseStorage {
const items: HNSWNounWithMetadata[] = []
for (const node of result.nodes) {
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
const metadata = await this.getNounMetadata(node.id)
if (!metadata) continue
// Apply filters if provided
if (options.filter) {
@ -1083,7 +1083,7 @@ export class GcsStorage extends BaseStorage {
vector: [...node.vector],
connections: new Map(node.connections),
level: node.level || 0,
metadata: metadata
metadata: (metadata || {}) as NounMetadata // Empty if none
}
items.push(nounWithMetadata)
}

11
src/storage/adapters/memoryStorage.ts
View file

@ -204,8 +204,8 @@ export class MemoryStorage extends BaseStorage {
if (!noun) continue
// Get metadata from separate storage
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
const metadata = await this.getNounMetadata(id)
if (!metadata) continue // Skip if no metadata
// v4.0.0: Create HNSWNounWithMetadata with metadata field
const nounWithMetadata: HNSWNounWithMetadata = {
@ -213,7 +213,7 @@ export class MemoryStorage extends BaseStorage {
vector: [...noun.vector],
connections: new Map(),
level: noun.level || 0,
metadata: metadata // Include metadata field
metadata: (metadata || {}) as NounMetadata // Include metadata field (empty if none)
}
// Copy connections
@ -466,8 +466,9 @@ export class MemoryStorage extends BaseStorage {
if (!hnswVerb) continue
// Get metadata from separate storage
// FIX v4.7.4: Don't skip verbs without metadata - metadata is optional in v4.0.0
// Core fields (verb, sourceId, targetId) are in HNSWVerb itself
const metadata = await this.getVerbMetadata(id)
if (!metadata) continue // Skip if no metadata
// v4.0.0: Create HNSWVerbWithMetadata with metadata field
const verbWithMetadata: HNSWVerbWithMetadata = {
@ -480,8 +481,8 @@ export class MemoryStorage extends BaseStorage {
sourceId: hnswVerb.sourceId,
targetId: hnswVerb.targetId,
// Metadata field
metadata: metadata
// Metadata field (empty if none)
metadata: metadata || {}
}
// Copy connections

13
src/storage/adapters/opfsStorage.ts
View file

@ -1686,11 +1686,11 @@ export class OPFSStorage extends BaseStorage {
const noun = await this.getNoun_internal(id)
if (noun) {
// Load metadata for filtering and combining
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
const metadata = await this.getNounMetadata(id)
if (!metadata) continue
// Apply filters if provided
if (options.filter) {
if (options.filter && metadata) {
// Filter by noun type
if (options.filter.nounType) {
const nounTypes = Array.isArray(options.filter.nounType)
@ -1730,7 +1730,7 @@ export class OPFSStorage extends BaseStorage {
vector: [...noun.vector],
connections: new Map(noun.connections),
level: noun.level || 0,
metadata: metadata
metadata: (metadata || {}) as NounMetadata // Empty if none
}
items.push(nounWithMetadata)
@ -1819,11 +1819,12 @@ export class OPFSStorage extends BaseStorage {
const hnswVerb = await this.getVerb_internal(id)
if (hnswVerb) {
// Load metadata for filtering and combining
// FIX v4.7.4: Don't skip verbs without metadata - metadata is optional in v4.0.0
// Core fields (verb, sourceId, targetId) are in HNSWVerb itself
const metadata = await this.getVerbMetadata(id)
if (!metadata) continue
// Apply filters if provided
if (options.filter) {
if (options.filter && metadata) {
// Filter by verb type
// v4.0.0: verb field is in HNSWVerb structure (NOT in metadata)
if (options.filter.verbType) {
@ -1888,7 +1889,7 @@ export class OPFSStorage extends BaseStorage {
verb: hnswVerb.verb,
sourceId: hnswVerb.sourceId,
targetId: hnswVerb.targetId,
metadata: metadata
metadata: (metadata || {}) as VerbMetadata // Empty if none
}
items.push(verbWithMetadata)

6
src/storage/adapters/r2Storage.ts
View file

@ -1190,11 +1190,11 @@ export class R2Storage extends BaseStorage {
const noun = await this.getNoun_internal(id)
if (noun) {
// v4.0.0: Load metadata and combine with noun to create HNSWNounWithMetadata
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
const metadata = await this.getNounMetadata(id)
if (!metadata) continue
// Apply filters if provided
if (options.filter) {
if (options.filter && metadata) {
// Filter by noun type
if (options.filter.nounType) {
const nounTypes = Array.isArray(options.filter.nounType)
@ -1234,7 +1234,7 @@ export class R2Storage extends BaseStorage {
vector: [...noun.vector],
connections: new Map(noun.connections),
level: noun.level || 0,
metadata: metadata
metadata: (metadata || {}) as NounMetadata // Empty if none
}
items.push(nounWithMetadata)

6
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -3683,11 +3683,11 @@ export class S3CompatibleStorage extends BaseStorage {
const nounsWithMetadata: HNSWNounWithMetadata[] = []
for (const node of result.nodes) {
// FIX v4.7.4: Don't skip nouns without metadata - metadata is optional in v4.0.0
const metadata = await this.getNounMetadata(node.id)
if (!metadata) continue
// Apply filters if provided
if (options.filter) {
if (options.filter && metadata) {
// Filter by noun type
if (options.filter.nounType) {
const nounTypes = Array.isArray(options.filter.nounType)
@ -3729,7 +3729,7 @@ export class S3CompatibleStorage extends BaseStorage {
vector: [...node.vector],
connections: new Map(node.connections),
level: node.level || 0,
metadata: metadata
metadata: (metadata || {}) as NounMetadata // Empty if none
}
nounsWithMetadata.push(nounWithMetadata)
}

49
src/storage/adapters/typeAwareStorageAdapter.ts
View file

@ -436,19 +436,28 @@ export class TypeAwareStorageAdapter extends BaseStorage {
// Check sourceId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
if (hnswVerb.sourceId !== sourceId) continue
// Load metadata separately
// Load metadata separately (optional in v4.0.0!)
// FIX: Don't skip verbs without metadata - metadata is optional!
// VFS relationships often have NO metadata (just verb/source/target)
const metadata = await this.getVerbMetadata(id)
if (!metadata) continue
// Create HNSWVerbWithMetadata (verbs don't have level field)
// Convert connections from plain object to Map<number, Set<string>>
const connectionsMap = new Map<number, Set<string>>()
if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
for (const [level, ids] of Object.entries(hnswVerb.connections)) {
connectionsMap.set(Number(level), new Set(ids as string[]))
}
}
const verbWithMetadata: HNSWVerbWithMetadata = {
id: hnswVerb.id,
vector: [...hnswVerb.vector],
connections: new Map(hnswVerb.connections),
connections: connectionsMap,
verb: hnswVerb.verb,
sourceId: hnswVerb.sourceId,
targetId: hnswVerb.targetId,
metadata: metadata
metadata: metadata || {} // Empty metadata if none exists
}
verbs.push(verbWithMetadata)
@ -485,19 +494,27 @@ export class TypeAwareStorageAdapter extends BaseStorage {
// Check targetId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
if (hnswVerb.targetId !== targetId) continue
// Load metadata separately
// Load metadata separately (optional in v4.0.0!)
// FIX: Don't skip verbs without metadata - metadata is optional!
const metadata = await this.getVerbMetadata(id)
if (!metadata) continue
// Create HNSWVerbWithMetadata (verbs don't have level field)
// Convert connections from plain object to Map<number, Set<string>>
const connectionsMap = new Map<number, Set<string>>()
if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
for (const [level, ids] of Object.entries(hnswVerb.connections)) {
connectionsMap.set(Number(level), new Set(ids as string[]))
}
}
const verbWithMetadata: HNSWVerbWithMetadata = {
id: hnswVerb.id,
vector: [...hnswVerb.vector],
connections: new Map(hnswVerb.connections),
connections: connectionsMap,
verb: hnswVerb.verb,
sourceId: hnswVerb.sourceId,
targetId: hnswVerb.targetId,
metadata: metadata
metadata: metadata || {} // Empty metadata if none exists
}
verbs.push(verbWithMetadata)
@ -530,19 +547,27 @@ export class TypeAwareStorageAdapter extends BaseStorage {
// Cache type from HNSWVerb for future O(1) retrievals
this.verbTypeCache.set(hnswVerb.id, hnswVerb.verb as VerbType)
// Load metadata separately
// Load metadata separately (optional in v4.0.0!)
// FIX: Don't skip verbs without metadata - metadata is optional!
const metadata = await this.getVerbMetadata(hnswVerb.id)
if (!metadata) continue
// Create HNSWVerbWithMetadata (verbs don't have level field)
// Convert connections from plain object to Map<number, Set<string>>
const connectionsMap = new Map<number, Set<string>>()
if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
for (const [level, ids] of Object.entries(hnswVerb.connections)) {
connectionsMap.set(Number(level), new Set(ids as string[]))
}
}
const verbWithMetadata: HNSWVerbWithMetadata = {
id: hnswVerb.id,
vector: [...hnswVerb.vector],
connections: new Map(hnswVerb.connections),
connections: connectionsMap,
verb: hnswVerb.verb,
sourceId: hnswVerb.sourceId,
targetId: hnswVerb.targetId,
metadata: metadata
metadata: metadata || {} // Empty metadata if none exists
}
verbs.push(verbWithMetadata)