soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: v4.8.1 - Fix 11-version VFS bug (metadata skip + TypeAware performance)

Browse source 
CRITICAL BUG FIX: FileSystemStorage was skipping verbs without metadata files,
causing getVerbsBySource() to return 0 results despite 601 relationships existing.

Root Cause:
- Workshop data had 601 verb vector files with 0 metadata files
- FileSystemStorage.getVerbsWithPagination() skipped verbs if metadata === null
- This bug survived 11 versions (v4.5.1 through v4.8.0)

Fixes Applied:

1. FileSystemStorage (2 locations):
   - Line 1391-1406: Remove metadata skip in getVerbsWithPagination()
   - Line 2427-2442: Remove metadata skip in streaming method
   - Use (metadata || {}) pattern like other adapters

2. TypeAwareStorage (2 methods):
   - getVerbsBySource_internal: Delegate to underlying storage (was O(total_verbs))
   - getVerbsByTarget_internal: Delegate to underlying storage
   - Fixes catastrophic performance bug (scanned all 40 types + all files)

Verification:
- Built successfully with TypeScript
- All 25 augmentation tests passing
- Tested against Workshop's 601 verb dataset
- Result: 0 verbs → 2 verbs returned ✓

Impact:
- VFS relationships now queryable without metadata files
- TypeAware performance: O(total_verbs) → O(matching_verbs)
- Compatible with existing data (backward compatible)
This commit is contained in:
David Snelling 2025-10-28 09:12:09 -07:00
parent 26204a7d67
commit dbb827a560
2 changed files with 24 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

20
src/storage/adapters/fileSystemStorage.ts
View file

@ -1388,11 +1388,9 @@ export class FileSystemStorage extends BaseStorage {
// Get metadata which contains the actual verb information
const metadata = await this.getVerbMetadata(id)
// v4.0.0: No fallbacks - skip verbs without metadata
if (!metadata) {
console.warn(`Verb ${id} has no metadata, skipping`)
continue
}
// v4.8.1: Don't skip verbs without metadata - metadata is optional
// FIX: This was the root cause of the VFS bug (11 versions)
// Verbs can exist without metadata files (e.g., from imports/migrations)
// Convert connections Map to proper format if needed
let connections = edge.connections
@ -1405,7 +1403,7 @@ export class FileSystemStorage extends BaseStorage {
}
// v4.8.0: Extract standard fields from metadata to top-level
const metadataObj = metadata as VerbMetadata
const metadataObj = (metadata || {}) as VerbMetadata
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj
const verbWithMetadata: HNSWVerbWithMetadata = {
@ -2426,11 +2424,9 @@ export class FileSystemStorage extends BaseStorage {
const edge = JSON.parse(data)
const metadata = await this.getVerbMetadata(id)
// v4.0.0: No fallbacks - skip verbs without metadata
if (!metadata) {
processedCount++
return true // continue, skip this verb
}
// v4.8.1: Don't skip verbs without metadata - metadata is optional
// FIX: This was the root cause of the VFS bug (11 versions)
// Verbs can exist without metadata files (e.g., from imports/migrations)
// Convert connections if needed
let connections = edge.connections
@ -2443,7 +2439,7 @@ export class FileSystemStorage extends BaseStorage {
}
// v4.8.0: Extract standard fields from metadata to top-level
const metadataObj = metadata as VerbMetadata
const metadataObj = (metadata || {}) as VerbMetadata
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj
const verbWithMetadata: HNSWVerbWithMetadata = {

141
src/storage/adapters/typeAwareStorageAdapter.ts
View file

@ -415,138 +415,29 @@ export class TypeAwareStorageAdapter extends BaseStorage {
* Get verbs by source
*/
protected async getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]> {
// Need to search across all verb types
// TODO: Optimize with metadata index in Phase 1b
const verbs: HNSWVerbWithMetadata[] = []
// v4.8.1 PERFORMANCE FIX: Delegate to underlying storage instead of scanning all files
// Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files
// This was the root cause of the 11-version VFS bug (timeouts/zero results)
//
// Underlying storage adapters have optimized implementations:
// - FileSystemStorage: Uses getVerbsWithPagination with sourceId filter
// - GcsStorage: Uses batch queries with prefix filtering
// - S3Storage: Uses listObjects with sourceId-based filtering
//
// Phase 1b TODO: Add graph adjacency index query for O(1) lookups:
// const verbIds = await this.graphIndex?.getOutgoingEdges(sourceId) || []
// return Promise.all(verbIds.map(id => this.getVerb(id)))
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
const type = TypeUtils.getVerbFromIndex(i)
const prefix = `entities/verbs/${type}/vectors/`
const paths = await this.u.listObjectsUnderPath(prefix)
for (const path of paths) {
try {
const id = path.split('/').pop()?.replace('.json', '')
if (!id) continue
// Load the HNSWVerb
const hnswVerb = await this.u.readObjectFromPath(path)
if (!hnswVerb) continue
// Check sourceId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
if (hnswVerb.sourceId !== sourceId) continue
// Load metadata separately (optional in v4.0.0!)
// FIX: Don't skip verbs without metadata - metadata is optional!
// VFS relationships often have NO metadata (just verb/source/target)
const metadata = await this.getVerbMetadata(id)
// Create HNSWVerbWithMetadata (verbs don't have level field)
// Convert connections from plain object to Map<number, Set<string>>
const connectionsMap = new Map<number, Set<string>>()
if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
for (const [level, ids] of Object.entries(hnswVerb.connections)) {
connectionsMap.set(Number(level), new Set(ids as string[]))
}
}
// v4.8.0: Extract standard fields from metadata to top-level
const metadataObj = (metadata || {}) as VerbMetadata
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj
const verbWithMetadata: HNSWVerbWithMetadata = {
id: hnswVerb.id,
vector: [...hnswVerb.vector],
connections: connectionsMap,
verb: hnswVerb.verb,
sourceId: hnswVerb.sourceId,
targetId: hnswVerb.targetId,
createdAt: (createdAt as number) || Date.now(),
updatedAt: (updatedAt as number) || Date.now(),
confidence: confidence as number | undefined,
weight: weight as number | undefined,
service: service as string | undefined,
data: data as Record<string, any> | undefined,
createdBy,
metadata: customMetadata
}
verbs.push(verbWithMetadata)
} catch (error) {
// Continue searching
}
}
}
return verbs
return this.underlying.getVerbsBySource(sourceId)
}
/**
* Get verbs by target
*/
protected async getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]> {
// Similar to getVerbsBySource_internal
const verbs: HNSWVerbWithMetadata[] = []
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
const type = TypeUtils.getVerbFromIndex(i)
const prefix = `entities/verbs/${type}/vectors/`
const paths = await this.u.listObjectsUnderPath(prefix)
for (const path of paths) {
try {
const id = path.split('/').pop()?.replace('.json', '')
if (!id) continue
// Load the HNSWVerb
const hnswVerb = await this.u.readObjectFromPath(path)
if (!hnswVerb) continue
// Check targetId from HNSWVerb (v4.0.0: core fields are in HNSWVerb)
if (hnswVerb.targetId !== targetId) continue
// Load metadata separately (optional in v4.0.0!)
// FIX: Don't skip verbs without metadata - metadata is optional!
const metadata = await this.getVerbMetadata(id)
// Create HNSWVerbWithMetadata (verbs don't have level field)
// Convert connections from plain object to Map<number, Set<string>>
const connectionsMap = new Map<number, Set<string>>()
if (hnswVerb.connections && typeof hnswVerb.connections === 'object') {
for (const [level, ids] of Object.entries(hnswVerb.connections)) {
connectionsMap.set(Number(level), new Set(ids as string[]))
}
}
// v4.8.0: Extract standard fields from metadata to top-level
const metadataObj = (metadata || {}) as VerbMetadata
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj
const verbWithMetadata: HNSWVerbWithMetadata = {
id: hnswVerb.id,
vector: [...hnswVerb.vector],
connections: connectionsMap,
verb: hnswVerb.verb,
sourceId: hnswVerb.sourceId,
targetId: hnswVerb.targetId,
createdAt: (createdAt as number) || Date.now(),
updatedAt: (updatedAt as number) || Date.now(),
confidence: confidence as number | undefined,
weight: weight as number | undefined,
service: service as string | undefined,
data: data as Record<string, any> | undefined,
createdBy,
metadata: customMetadata
}
verbs.push(verbWithMetadata)
} catch (error) {
// Continue
}
}
}
return verbs
// v4.8.1 PERFORMANCE FIX: Delegate to underlying storage (same as getVerbsBySource fix)
// Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files
return this.underlying.getVerbsByTarget(targetId)
}
/**