diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 179be002..c2daad8b 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -1388,11 +1388,9 @@ export class FileSystemStorage extends BaseStorage { // Get metadata which contains the actual verb information const metadata = await this.getVerbMetadata(id) - // v4.0.0: No fallbacks - skip verbs without metadata - if (!metadata) { - console.warn(`Verb ${id} has no metadata, skipping`) - continue - } + // v4.8.1: Don't skip verbs without metadata - metadata is optional + // FIX: This was the root cause of the VFS bug (11 versions) + // Verbs can exist without metadata files (e.g., from imports/migrations) // Convert connections Map to proper format if needed let connections = edge.connections @@ -1405,7 +1403,7 @@ export class FileSystemStorage extends BaseStorage { } // v4.8.0: Extract standard fields from metadata to top-level - const metadataObj = metadata as VerbMetadata + const metadataObj = (metadata || {}) as VerbMetadata const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj const verbWithMetadata: HNSWVerbWithMetadata = { @@ -2426,11 +2424,9 @@ export class FileSystemStorage extends BaseStorage { const edge = JSON.parse(data) const metadata = await this.getVerbMetadata(id) - // v4.0.0: No fallbacks - skip verbs without metadata - if (!metadata) { - processedCount++ - return true // continue, skip this verb - } + // v4.8.1: Don't skip verbs without metadata - metadata is optional + // FIX: This was the root cause of the VFS bug (11 versions) + // Verbs can exist without metadata files (e.g., from imports/migrations) // Convert connections if needed let connections = edge.connections @@ -2443,7 +2439,7 @@ export class FileSystemStorage extends BaseStorage { } // v4.8.0: Extract standard fields from metadata to top-level - const metadataObj = metadata as VerbMetadata + const metadataObj = (metadata || {}) as VerbMetadata const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj const verbWithMetadata: HNSWVerbWithMetadata = { diff --git a/src/storage/adapters/typeAwareStorageAdapter.ts b/src/storage/adapters/typeAwareStorageAdapter.ts index 56de3613..cf3456fd 100644 --- a/src/storage/adapters/typeAwareStorageAdapter.ts +++ b/src/storage/adapters/typeAwareStorageAdapter.ts @@ -415,138 +415,29 @@ export class TypeAwareStorageAdapter extends BaseStorage { * Get verbs by source */ protected async getVerbsBySource_internal(sourceId: string): Promise { - // Need to search across all verb types - // TODO: Optimize with metadata index in Phase 1b - const verbs: HNSWVerbWithMetadata[] = [] + // v4.8.1 PERFORMANCE FIX: Delegate to underlying storage instead of scanning all files + // Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files + // This was the root cause of the 11-version VFS bug (timeouts/zero results) + // + // Underlying storage adapters have optimized implementations: + // - FileSystemStorage: Uses getVerbsWithPagination with sourceId filter + // - GcsStorage: Uses batch queries with prefix filtering + // - S3Storage: Uses listObjects with sourceId-based filtering + // + // Phase 1b TODO: Add graph adjacency index query for O(1) lookups: + // const verbIds = await this.graphIndex?.getOutgoingEdges(sourceId) || [] + // return Promise.all(verbIds.map(id => this.getVerb(id))) - for (let i = 0; i < VERB_TYPE_COUNT; i++) { - const type = TypeUtils.getVerbFromIndex(i) - const prefix = `entities/verbs/${type}/vectors/` - const paths = await this.u.listObjectsUnderPath(prefix) - - for (const path of paths) { - try { - const id = path.split('/').pop()?.replace('.json', '') - if (!id) continue - - // Load the HNSWVerb - const hnswVerb = await this.u.readObjectFromPath(path) - if (!hnswVerb) continue - - // Check sourceId from HNSWVerb (v4.0.0: core fields are in HNSWVerb) - if (hnswVerb.sourceId !== sourceId) continue - - // Load metadata separately (optional in v4.0.0!) - // FIX: Don't skip verbs without metadata - metadata is optional! - // VFS relationships often have NO metadata (just verb/source/target) - const metadata = await this.getVerbMetadata(id) - - // Create HNSWVerbWithMetadata (verbs don't have level field) - // Convert connections from plain object to Map> - const connectionsMap = new Map>() - if (hnswVerb.connections && typeof hnswVerb.connections === 'object') { - for (const [level, ids] of Object.entries(hnswVerb.connections)) { - connectionsMap.set(Number(level), new Set(ids as string[])) - } - } - - // v4.8.0: Extract standard fields from metadata to top-level - const metadataObj = (metadata || {}) as VerbMetadata - const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj - - const verbWithMetadata: HNSWVerbWithMetadata = { - id: hnswVerb.id, - vector: [...hnswVerb.vector], - connections: connectionsMap, - verb: hnswVerb.verb, - sourceId: hnswVerb.sourceId, - targetId: hnswVerb.targetId, - createdAt: (createdAt as number) || Date.now(), - updatedAt: (updatedAt as number) || Date.now(), - confidence: confidence as number | undefined, - weight: weight as number | undefined, - service: service as string | undefined, - data: data as Record | undefined, - createdBy, - metadata: customMetadata - } - - verbs.push(verbWithMetadata) - } catch (error) { - // Continue searching - } - } - } - - return verbs + return this.underlying.getVerbsBySource(sourceId) } /** * Get verbs by target */ protected async getVerbsByTarget_internal(targetId: string): Promise { - // Similar to getVerbsBySource_internal - const verbs: HNSWVerbWithMetadata[] = [] - - for (let i = 0; i < VERB_TYPE_COUNT; i++) { - const type = TypeUtils.getVerbFromIndex(i) - const prefix = `entities/verbs/${type}/vectors/` - const paths = await this.u.listObjectsUnderPath(prefix) - - for (const path of paths) { - try { - const id = path.split('/').pop()?.replace('.json', '') - if (!id) continue - - // Load the HNSWVerb - const hnswVerb = await this.u.readObjectFromPath(path) - if (!hnswVerb) continue - - // Check targetId from HNSWVerb (v4.0.0: core fields are in HNSWVerb) - if (hnswVerb.targetId !== targetId) continue - - // Load metadata separately (optional in v4.0.0!) - // FIX: Don't skip verbs without metadata - metadata is optional! - const metadata = await this.getVerbMetadata(id) - - // Create HNSWVerbWithMetadata (verbs don't have level field) - // Convert connections from plain object to Map> - const connectionsMap = new Map>() - if (hnswVerb.connections && typeof hnswVerb.connections === 'object') { - for (const [level, ids] of Object.entries(hnswVerb.connections)) { - connectionsMap.set(Number(level), new Set(ids as string[])) - } - } - - // v4.8.0: Extract standard fields from metadata to top-level - const metadataObj = (metadata || {}) as VerbMetadata - const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj - - const verbWithMetadata: HNSWVerbWithMetadata = { - id: hnswVerb.id, - vector: [...hnswVerb.vector], - connections: connectionsMap, - verb: hnswVerb.verb, - sourceId: hnswVerb.sourceId, - targetId: hnswVerb.targetId, - createdAt: (createdAt as number) || Date.now(), - updatedAt: (updatedAt as number) || Date.now(), - confidence: confidence as number | undefined, - weight: weight as number | undefined, - service: service as string | undefined, - data: data as Record | undefined, - createdBy, - metadata: customMetadata - } - - verbs.push(verbWithMetadata) - } catch (error) { - // Continue - } - } - } - - return verbs + // v4.8.1 PERFORMANCE FIX: Delegate to underlying storage (same as getVerbsBySource fix) + // Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files + return this.underlying.getVerbsByTarget(targetId) } /**