fix: combine vector and metadata in getNoun/getVerb internal methods

Fixed critical bug in 2-file architecture where getNoun_internal() and
getVerb_internal() only returned vector data without metadata, causing
brain.get() to return null despite data being correctly saved.

Changes:
- Updated all 5 storage adapters (GCS, S3, FileSystem, Memory, OPFS)
- Fixed getNoun_internal() to fetch and combine vector + metadata
- Fixed getVerb_internal() to fetch and combine vector + metadata
- Added metadata field to HNSWVerb interface for consistency

The 2-file architecture now works correctly for both read and write operations.
This commit is contained in:
David Snelling 2025-10-10 16:51:59 -07:00
parent 30063d440a
commit cb1e37c0e8
6 changed files with 140 additions and 30 deletions

View file

@ -94,6 +94,7 @@ export interface HNSWVerb {
id: string id: string
vector: Vector vector: Vector
connections: Map<number, Set<string>> // level -> set of connected verb ids connections: Map<number, Set<string>> // level -> set of connected verb ids
metadata?: any // Optional metadata for the verb (2-file system)
} }
/** /**

View file

@ -1043,10 +1043,24 @@ export class FileSystemStorage extends BaseStorage {
} }
/** /**
* Get a noun from storage * Get a noun from storage (internal implementation)
* Combines vector data from getNode() with metadata from getNounMetadata()
*/ */
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> { protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
return this.getNode(id) // Get vector data (lightweight)
const node = await this.getNode(id)
if (!node) {
return null
}
// Get metadata (entity data in 2-file system)
const metadata = await this.getNounMetadata(id)
// Combine into complete noun object
return {
...node,
metadata: metadata || {}
}
} }
@ -1074,10 +1088,24 @@ export class FileSystemStorage extends BaseStorage {
} }
/** /**
* Get a verb from storage * Get a verb from storage (internal implementation)
* Combines vector data from getEdge() with metadata from getVerbMetadata()
*/ */
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> { protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
return this.getEdge(id) // Get vector data (lightweight)
const edge = await this.getEdge(id)
if (!edge) {
return null
}
// Get metadata (relationship data in 2-file system)
const metadata = await this.getVerbMetadata(id)
// Combine into complete verb object
return {
...edge,
metadata: metadata || {}
}
} }

View file

@ -482,9 +482,23 @@ export class GcsStorage extends BaseStorage {
/** /**
* Get a noun from storage (internal implementation) * Get a noun from storage (internal implementation)
* Combines vector data from getNode() with metadata from getNounMetadata()
*/ */
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> { protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
return this.getNode(id) // Get vector data (lightweight)
const node = await this.getNode(id)
if (!node) {
return null
}
// Get metadata (entity data in 2-file system)
const metadata = await this.getNounMetadata(id)
// Combine into complete noun object
return {
...node,
metadata: metadata || {}
}
} }
/** /**
@ -798,9 +812,23 @@ export class GcsStorage extends BaseStorage {
/** /**
* Get a verb from storage (internal implementation) * Get a verb from storage (internal implementation)
* Combines vector data from getEdge() with metadata from getVerbMetadata()
*/ */
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> { protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
return this.getEdge(id) // Get vector data (lightweight)
const edge = await this.getEdge(id)
if (!edge) {
return null
}
// Get metadata (relationship data in 2-file system)
const metadata = await this.getVerbMetadata(id)
// Combine into complete verb object
return {
...edge,
metadata: metadata || {}
}
} }
/** /**

View file

@ -78,7 +78,8 @@ export class MemoryStorage extends BaseStorage {
} }
/** /**
* Get a noun from storage * Get a noun from storage (internal implementation)
* Combines vector data from nouns map with metadata from getNounMetadata()
*/ */
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> { protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
// Get the noun directly from the nouns map // Get the noun directly from the nouns map
@ -90,14 +91,11 @@ export class MemoryStorage extends BaseStorage {
} }
// Return a deep copy to avoid reference issues // Return a deep copy to avoid reference issues
// CRITICAL: Only return lightweight vector data (no metadata)
// Metadata is retrieved separately via getNounMetadata() (2-file system)
const nounCopy: HNSWNoun = { const nounCopy: HNSWNoun = {
id: noun.id, id: noun.id,
vector: [...noun.vector], vector: [...noun.vector],
connections: new Map(), connections: new Map(),
level: noun.level || 0 level: noun.level || 0
// NO metadata field - retrieved separately for scalability
} }
// Copy connections // Copy connections
@ -105,7 +103,14 @@ export class MemoryStorage extends BaseStorage {
nounCopy.connections.set(level, new Set(connections)) nounCopy.connections.set(level, new Set(connections))
} }
return nounCopy // Get metadata (entity data in 2-file system)
const metadata = await this.getNounMetadata(id)
// Combine into complete noun object
return {
...nounCopy,
metadata: metadata || {}
}
} }
/** /**
@ -303,7 +308,8 @@ export class MemoryStorage extends BaseStorage {
} }
/** /**
* Get a verb from storage * Get a verb from storage (internal implementation)
* Combines vector data from verbs map with metadata from getVerbMetadata()
*/ */
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> { protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
// Get the verb directly from the verbs map // Get the verb directly from the verbs map
@ -314,18 +320,6 @@ export class MemoryStorage extends BaseStorage {
return null return null
} }
// Create default timestamp if not present
const defaultTimestamp = {
seconds: Math.floor(Date.now() / 1000),
nanoseconds: (Date.now() % 1000) * 1000000
}
// Create default createdBy if not present
const defaultCreatedBy = {
augmentation: 'unknown',
version: '1.0'
}
// Return a deep copy of the HNSWVerb // Return a deep copy of the HNSWVerb
const verbCopy: HNSWVerb = { const verbCopy: HNSWVerb = {
id: verb.id, id: verb.id,
@ -338,7 +332,14 @@ export class MemoryStorage extends BaseStorage {
verbCopy.connections.set(level, new Set(connections)) verbCopy.connections.set(level, new Set(connections))
} }
return verbCopy // Get metadata (relationship data in 2-file system)
const metadata = await this.getVerbMetadata(id)
// Combine into complete verb object
return {
...verbCopy,
metadata: metadata || {}
}
} }
/** /**

View file

@ -237,7 +237,8 @@ export class OPFSStorage extends BaseStorage {
} }
/** /**
* Get a noun from storage * Get a noun from storage (internal implementation)
* Combines vector data from file with metadata from getNounMetadata()
*/ */
protected async getNoun_internal( protected async getNoun_internal(
id: string id: string
@ -265,12 +266,21 @@ export class OPFSStorage extends BaseStorage {
connections.set(Number(level), new Set(nounIds as string[])) connections.set(Number(level), new Set(nounIds as string[]))
} }
return { const node = {
id: data.id, id: data.id,
vector: data.vector, vector: data.vector,
connections, connections,
level: data.level || 0 level: data.level || 0
} }
// Get metadata (entity data in 2-file system)
const metadata = await this.getNounMetadata(id)
// Combine into complete noun object
return {
...node,
metadata: metadata || {}
}
} catch (error) { } catch (error) {
// Noun not found or other error // Noun not found or other error
return null return null
@ -427,9 +437,23 @@ export class OPFSStorage extends BaseStorage {
/** /**
* Get a verb from storage (internal implementation) * Get a verb from storage (internal implementation)
* Combines vector data from getEdge() with metadata from getVerbMetadata()
*/ */
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> { protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
return this.getEdge(id) // Get vector data (lightweight)
const edge = await this.getEdge(id)
if (!edge) {
return null
}
// Get metadata (relationship data in 2-file system)
const metadata = await this.getVerbMetadata(id)
// Combine into complete verb object
return {
...edge,
metadata: metadata || {}
}
} }
/** /**

View file

@ -1048,9 +1048,23 @@ export class S3CompatibleStorage extends BaseStorage {
/** /**
* Get a noun from storage (internal implementation) * Get a noun from storage (internal implementation)
* Combines vector data from getNode() with metadata from getNounMetadata()
*/ */
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> { protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
return this.getNode(id) // Get vector data (lightweight)
const node = await this.getNode(id)
if (!node) {
return null
}
// Get metadata (entity data in 2-file system)
const metadata = await this.getNounMetadata(id)
// Combine into complete noun object
return {
...node,
metadata: metadata || {}
}
} }
/** /**
@ -1507,9 +1521,23 @@ export class S3CompatibleStorage extends BaseStorage {
/** /**
* Get a verb from storage (internal implementation) * Get a verb from storage (internal implementation)
* Combines vector data from getEdge() with metadata from getVerbMetadata()
*/ */
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> { protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
return this.getEdge(id) // Get vector data (lightweight)
const edge = await this.getEdge(id)
if (!edge) {
return null
}
// Get metadata (relationship data in 2-file system)
const metadata = await this.getVerbMetadata(id)
// Combine into complete verb object
return {
...edge,
metadata: metadata || {}
}
} }
/** /**