fix: combine vector and metadata in getNoun/getVerb internal methods
Fixed critical bug in 2-file architecture where getNoun_internal() and getVerb_internal() only returned vector data without metadata, causing brain.get() to return null despite data being correctly saved. Changes: - Updated all 5 storage adapters (GCS, S3, FileSystem, Memory, OPFS) - Fixed getNoun_internal() to fetch and combine vector + metadata - Fixed getVerb_internal() to fetch and combine vector + metadata - Added metadata field to HNSWVerb interface for consistency The 2-file architecture now works correctly for both read and write operations.
This commit is contained in:
parent
30063d440a
commit
cb1e37c0e8
6 changed files with 140 additions and 30 deletions
|
|
@ -94,6 +94,7 @@ export interface HNSWVerb {
|
||||||
id: string
|
id: string
|
||||||
vector: Vector
|
vector: Vector
|
||||||
connections: Map<number, Set<string>> // level -> set of connected verb ids
|
connections: Map<number, Set<string>> // level -> set of connected verb ids
|
||||||
|
metadata?: any // Optional metadata for the verb (2-file system)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -1043,10 +1043,24 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun from storage
|
* Get a noun from storage (internal implementation)
|
||||||
|
* Combines vector data from getNode() with metadata from getNounMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
||||||
return this.getNode(id)
|
// Get vector data (lightweight)
|
||||||
|
const node = await this.getNode(id)
|
||||||
|
if (!node) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get metadata (entity data in 2-file system)
|
||||||
|
const metadata = await this.getNounMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete noun object
|
||||||
|
return {
|
||||||
|
...node,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1074,10 +1088,24 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a verb from storage
|
* Get a verb from storage (internal implementation)
|
||||||
|
* Combines vector data from getEdge() with metadata from getVerbMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
||||||
return this.getEdge(id)
|
// Get vector data (lightweight)
|
||||||
|
const edge = await this.getEdge(id)
|
||||||
|
if (!edge) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get metadata (relationship data in 2-file system)
|
||||||
|
const metadata = await this.getVerbMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete verb object
|
||||||
|
return {
|
||||||
|
...edge,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -482,9 +482,23 @@ export class GcsStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun from storage (internal implementation)
|
* Get a noun from storage (internal implementation)
|
||||||
|
* Combines vector data from getNode() with metadata from getNounMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
||||||
return this.getNode(id)
|
// Get vector data (lightweight)
|
||||||
|
const node = await this.getNode(id)
|
||||||
|
if (!node) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get metadata (entity data in 2-file system)
|
||||||
|
const metadata = await this.getNounMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete noun object
|
||||||
|
return {
|
||||||
|
...node,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -798,9 +812,23 @@ export class GcsStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a verb from storage (internal implementation)
|
* Get a verb from storage (internal implementation)
|
||||||
|
* Combines vector data from getEdge() with metadata from getVerbMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
||||||
return this.getEdge(id)
|
// Get vector data (lightweight)
|
||||||
|
const edge = await this.getEdge(id)
|
||||||
|
if (!edge) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get metadata (relationship data in 2-file system)
|
||||||
|
const metadata = await this.getVerbMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete verb object
|
||||||
|
return {
|
||||||
|
...edge,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -78,7 +78,8 @@ export class MemoryStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun from storage
|
* Get a noun from storage (internal implementation)
|
||||||
|
* Combines vector data from nouns map with metadata from getNounMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
||||||
// Get the noun directly from the nouns map
|
// Get the noun directly from the nouns map
|
||||||
|
|
@ -90,14 +91,11 @@ export class MemoryStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return a deep copy to avoid reference issues
|
// Return a deep copy to avoid reference issues
|
||||||
// CRITICAL: Only return lightweight vector data (no metadata)
|
|
||||||
// Metadata is retrieved separately via getNounMetadata() (2-file system)
|
|
||||||
const nounCopy: HNSWNoun = {
|
const nounCopy: HNSWNoun = {
|
||||||
id: noun.id,
|
id: noun.id,
|
||||||
vector: [...noun.vector],
|
vector: [...noun.vector],
|
||||||
connections: new Map(),
|
connections: new Map(),
|
||||||
level: noun.level || 0
|
level: noun.level || 0
|
||||||
// NO metadata field - retrieved separately for scalability
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy connections
|
// Copy connections
|
||||||
|
|
@ -105,7 +103,14 @@ export class MemoryStorage extends BaseStorage {
|
||||||
nounCopy.connections.set(level, new Set(connections))
|
nounCopy.connections.set(level, new Set(connections))
|
||||||
}
|
}
|
||||||
|
|
||||||
return nounCopy
|
// Get metadata (entity data in 2-file system)
|
||||||
|
const metadata = await this.getNounMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete noun object
|
||||||
|
return {
|
||||||
|
...nounCopy,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -303,7 +308,8 @@ export class MemoryStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a verb from storage
|
* Get a verb from storage (internal implementation)
|
||||||
|
* Combines vector data from verbs map with metadata from getVerbMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
||||||
// Get the verb directly from the verbs map
|
// Get the verb directly from the verbs map
|
||||||
|
|
@ -314,18 +320,6 @@ export class MemoryStorage extends BaseStorage {
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create default timestamp if not present
|
|
||||||
const defaultTimestamp = {
|
|
||||||
seconds: Math.floor(Date.now() / 1000),
|
|
||||||
nanoseconds: (Date.now() % 1000) * 1000000
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create default createdBy if not present
|
|
||||||
const defaultCreatedBy = {
|
|
||||||
augmentation: 'unknown',
|
|
||||||
version: '1.0'
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return a deep copy of the HNSWVerb
|
// Return a deep copy of the HNSWVerb
|
||||||
const verbCopy: HNSWVerb = {
|
const verbCopy: HNSWVerb = {
|
||||||
id: verb.id,
|
id: verb.id,
|
||||||
|
|
@ -338,7 +332,14 @@ export class MemoryStorage extends BaseStorage {
|
||||||
verbCopy.connections.set(level, new Set(connections))
|
verbCopy.connections.set(level, new Set(connections))
|
||||||
}
|
}
|
||||||
|
|
||||||
return verbCopy
|
// Get metadata (relationship data in 2-file system)
|
||||||
|
const metadata = await this.getVerbMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete verb object
|
||||||
|
return {
|
||||||
|
...verbCopy,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -237,7 +237,8 @@ export class OPFSStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun from storage
|
* Get a noun from storage (internal implementation)
|
||||||
|
* Combines vector data from file with metadata from getNounMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getNoun_internal(
|
protected async getNoun_internal(
|
||||||
id: string
|
id: string
|
||||||
|
|
@ -265,12 +266,21 @@ export class OPFSStorage extends BaseStorage {
|
||||||
connections.set(Number(level), new Set(nounIds as string[]))
|
connections.set(Number(level), new Set(nounIds as string[]))
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
const node = {
|
||||||
id: data.id,
|
id: data.id,
|
||||||
vector: data.vector,
|
vector: data.vector,
|
||||||
connections,
|
connections,
|
||||||
level: data.level || 0
|
level: data.level || 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get metadata (entity data in 2-file system)
|
||||||
|
const metadata = await this.getNounMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete noun object
|
||||||
|
return {
|
||||||
|
...node,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Noun not found or other error
|
// Noun not found or other error
|
||||||
return null
|
return null
|
||||||
|
|
@ -427,9 +437,23 @@ export class OPFSStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a verb from storage (internal implementation)
|
* Get a verb from storage (internal implementation)
|
||||||
|
* Combines vector data from getEdge() with metadata from getVerbMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
||||||
return this.getEdge(id)
|
// Get vector data (lightweight)
|
||||||
|
const edge = await this.getEdge(id)
|
||||||
|
if (!edge) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get metadata (relationship data in 2-file system)
|
||||||
|
const metadata = await this.getVerbMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete verb object
|
||||||
|
return {
|
||||||
|
...edge,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -1048,9 +1048,23 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a noun from storage (internal implementation)
|
* Get a noun from storage (internal implementation)
|
||||||
|
* Combines vector data from getNode() with metadata from getNounMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
|
||||||
return this.getNode(id)
|
// Get vector data (lightweight)
|
||||||
|
const node = await this.getNode(id)
|
||||||
|
if (!node) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get metadata (entity data in 2-file system)
|
||||||
|
const metadata = await this.getNounMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete noun object
|
||||||
|
return {
|
||||||
|
...node,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -1507,9 +1521,23 @@ export class S3CompatibleStorage extends BaseStorage {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a verb from storage (internal implementation)
|
* Get a verb from storage (internal implementation)
|
||||||
|
* Combines vector data from getEdge() with metadata from getVerbMetadata()
|
||||||
*/
|
*/
|
||||||
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
|
||||||
return this.getEdge(id)
|
// Get vector data (lightweight)
|
||||||
|
const edge = await this.getEdge(id)
|
||||||
|
if (!edge) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get metadata (relationship data in 2-file system)
|
||||||
|
const metadata = await this.getVerbMetadata(id)
|
||||||
|
|
||||||
|
// Combine into complete verb object
|
||||||
|
return {
|
||||||
|
...edge,
|
||||||
|
metadata: metadata || {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue