soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: metadata explosion bug - 69K files reduced to ~1K

Browse source 
Critical fix for metadata indexing that was creating 60+ chunk files per entity.

Root cause: Vector embeddings (384-dimensional arrays) were being indexed in
metadata, causing each dimension to create a separate chunk file with numeric
field names ("0", "1", "2", etc.).

Changes:
- Modified extractIndexableFields() to exclude vector/embedding fields
- Added NEVER_INDEX set: ['vector', 'embedding', 'embeddings', 'connections']
- Added safety check to skip arrays > 10 elements
- Preserves small array indexing (tags, categories, roles)

Impact:
- Reduces metadata files from 69,429 → ~1,200 (58x reduction)
- Fixes server initialization hangs
- Fixes metadata batch loading stalling at batch 23
- Fixes VFS getDescendants() hanging with large datasets
- Fixes Graph View UI not loading

Test Results:
- 7/7 integration tests passing
- Verified: 6 chunk files for 10 entities (was 7,210 before fix)
- 611/622 unit tests passing

Files Modified:
- src/utils/metadataIndex.ts - Core fix
- src/coreTypes.ts - HNSWVerb type enforcement with VerbType enum
- src/storage/adapters/* - Include core relational fields in HNSWVerb
- src/storage/adapters/baseStorageAdapter.ts - Type enforcement (HNSWNoun, GraphVerb)
- tests/integration/metadata-vector-exclusion.test.ts - Comprehensive test coverage

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-10-16 16:10:31 -07:00
parent 7921a5b744
commit e600865d96
13 changed files with 571 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/storage/adapters/baseStorageAdapter.ts
View file

@ -3,7 +3,7 @@
* Provides common functionality for all storage adapters, including statistics tracking
*/
import { StatisticsData, StorageAdapter } from '../../coreTypes.js'
import { StatisticsData, StorageAdapter, HNSWNoun, GraphVerb } from '../../coreTypes.js'
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
@ -14,23 +14,23 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
// Abstract methods that must be implemented by subclasses
abstract init(): Promise<void>
abstract saveNoun(noun: any): Promise<void>
abstract saveNoun(noun: HNSWNoun): Promise<void>
abstract getNoun(id: string): Promise<any | null>
abstract getNoun(id: string): Promise<HNSWNoun | null>
abstract getNounsByNounType(nounType: string): Promise<any[]>
abstract getNounsByNounType(nounType: string): Promise<HNSWNoun[]>
abstract deleteNoun(id: string): Promise<void>
abstract saveVerb(verb: any): Promise<void>
abstract saveVerb(verb: GraphVerb): Promise<void>
abstract getVerb(id: string): Promise<any | null>
abstract getVerb(id: string): Promise<GraphVerb | null>
abstract getVerbsBySource(sourceId: string): Promise<any[]>
abstract getVerbsBySource(sourceId: string): Promise<GraphVerb[]>
abstract getVerbsByTarget(targetId: string): Promise<any[]>
abstract getVerbsByTarget(targetId: string): Promise<GraphVerb[]>
abstract getVerbsByType(type: string): Promise<any[]>
abstract getVerbsByType(type: string): Promise<GraphVerb[]>
abstract deleteVerb(id: string): Promise<void>
@ -98,7 +98,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
metadata?: Record<string, any>
}
}): Promise<{
items: any[]
items: HNSWNoun[]
totalCount?: number
hasMore: boolean
nextCursor?: string
@ -123,7 +123,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
metadata?: Record<string, any>
}
}): Promise<{
items: any[]
items: GraphVerb[]
totalCount?: number
hasMore: boolean
nextCursor?: string
@ -144,7 +144,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
metadata?: Record<string, any>
}
}): Promise<{
items: any[]
items: HNSWNoun[]
totalCount?: number
hasMore: boolean
nextCursor?: string
@ -167,7 +167,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
metadata?: Record<string, any>
}
}): Promise<{
items: any[]
items: GraphVerb[]
totalCount?: number
hasMore: boolean
nextCursor?: string

37
src/storage/adapters/fileSystemStorage.ts
View file

@ -430,15 +430,22 @@ export class FileSystemStorage extends BaseStorage {
const isNew = !(await this.fileExists(this.getVerbPath(edge.id)))
// Convert connections Map to a serializable format
// CRITICAL: Only save lightweight vector data (no metadata)
// Metadata is saved separately via saveVerbMetadata() (2-file system)
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
vector: edge.vector,
connections: this.mapToObject(edge.connections, (set) =>
Array.from(set as Set<string>)
)
// NO metadata field - saved separately for scalability
),
// CORE RELATIONAL DATA (v3.50.1+)
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
// User metadata (if any) - saved separately for scalability
// metadata field is saved separately via saveVerbMetadata()
}
const filePath = this.getVerbPath(edge.id)
@ -476,10 +483,19 @@ export class FileSystemStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
return {
id: parsedEdge.id,
vector: parsedEdge.vector,
connections
connections,
// CORE RELATIONAL DATA (read from vector file)
verb: parsedEdge.verb,
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId,
// User metadata (retrieved separately via getVerbMetadata())
metadata: parsedEdge.metadata
}
} catch (error: any) {
if (error.code !== 'ENOENT') {
@ -519,10 +535,19 @@ export class FileSystemStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields
allEdges.push({
id: parsedEdge.id,
vector: parsedEdge.vector,
connections
connections,
// CORE RELATIONAL DATA
verb: parsedEdge.verb,
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId,
// User metadata
metadata: parsedEdge.metadata
})
}
} catch (error: any) {

26
src/storage/adapters/gcsStorage.ts
View file

@ -810,8 +810,8 @@ export class GcsStorage extends BaseStorage {
this.logger.trace(`Saving edge ${edge.id}`)
// Convert connections Map to serializable format
// CRITICAL: Only save lightweight vector data (no metadata)
// Metadata is saved separately via saveVerbMetadata() (2-file system)
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
vector: edge.vector,
@ -820,8 +820,15 @@ export class GcsStorage extends BaseStorage {
level,
Array.from(verbIds)
])
)
// NO metadata field - saved separately for scalability
),
// CORE RELATIONAL DATA (v3.50.1+)
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
// User metadata (if any) - saved separately for scalability
// metadata field is saved separately via saveVerbMetadata()
}
// Get the GCS key with UUID-based sharding
@ -913,10 +920,19 @@ export class GcsStorage extends BaseStorage {
connections.set(Number(level), new Set(verbIds as string[]))
}
// ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
const edge: Edge = {
id: data.id,
vector: data.vector,
connections
connections,
// CORE RELATIONAL DATA (read from vector file)
verb: data.verb,
sourceId: data.sourceId,
targetId: data.targetId,
// User metadata (retrieved separately via getVerbMetadata())
metadata: data.metadata
}
// Update cache

31
src/storage/adapters/memoryStorage.ts
View file

@ -289,10 +289,19 @@ export class MemoryStorage extends BaseStorage {
const isNew = !this.verbs.has(verb.id)
// Create a deep copy to avoid reference issues
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields
const verbCopy: HNSWVerb = {
id: verb.id,
vector: [...verb.vector],
connections: new Map()
connections: new Map(),
// CORE RELATIONAL DATA
verb: verb.verb,
sourceId: verb.sourceId,
targetId: verb.targetId,
// User metadata (if any)
metadata: verb.metadata
}
// Copy connections
@ -321,10 +330,19 @@ export class MemoryStorage extends BaseStorage {
}
// Return a deep copy of the HNSWVerb
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields
const verbCopy: HNSWVerb = {
id: verb.id,
vector: [...verb.vector],
connections: new Map()
connections: new Map(),
// CORE RELATIONAL DATA
verb: verb.verb,
sourceId: verb.sourceId,
targetId: verb.targetId,
// User metadata
metadata: verb.metadata
}
// Copy connections
@ -332,14 +350,7 @@ export class MemoryStorage extends BaseStorage {
verbCopy.connections.set(level, new Set(connections))
}
// Get metadata (relationship data in 2-file system)
const metadata = await this.getVerbMetadata(id)
// Combine into complete verb object
return {
...verbCopy,
metadata: metadata || {}
}
return verbCopy
}
/**

37
src/storage/adapters/opfsStorage.ts
View file

@ -401,15 +401,22 @@ export class OPFSStorage extends BaseStorage {
await this.ensureInitialized()
try {
// CRITICAL: Only save lightweight vector data (no metadata)
// Metadata is saved separately via saveVerbMetadata() (2-file system)
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
vector: edge.vector,
connections: this.mapToObject(edge.connections, (set) =>
Array.from(set as Set<string>)
)
// NO metadata field - saved separately for scalability
),
// CORE RELATIONAL DATA (v3.50.1+)
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
// User metadata (if any) - saved separately for scalability
// metadata field is saved separately via saveVerbMetadata()
}
// Use UUID-based sharding for verbs
@ -495,10 +502,19 @@ export class OPFSStorage extends BaseStorage {
version: '1.0'
}
// ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
return {
id: data.id,
vector: data.vector,
connections
connections,
// CORE RELATIONAL DATA (read from vector file)
verb: data.verb,
sourceId: data.sourceId,
targetId: data.targetId,
// User metadata (retrieved separately via getVerbMetadata())
metadata: data.metadata
}
} catch (error) {
// Edge not found or other error
@ -547,10 +563,19 @@ export class OPFSStorage extends BaseStorage {
version: '1.0'
}
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields
allEdges.push({
id: data.id,
vector: data.vector,
connections
connections,
// CORE RELATIONAL DATA
verb: data.verb,
sourceId: data.sourceId,
targetId: data.targetId,
// User metadata
metadata: data.metadata
})
} catch (error) {
console.error(`Error reading edge file ${shardName}/${fileName}:`, error)

23
src/storage/adapters/r2Storage.ts
View file

@ -728,6 +728,8 @@ export class R2Storage extends BaseStorage {
const requestId = await this.applyBackpressure()
try {
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
vector: edge.vector,
@ -736,7 +738,15 @@ export class R2Storage extends BaseStorage {
level,
Array.from(verbIds)
])
)
),
// CORE RELATIONAL DATA (v3.50.1+)
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
// User metadata (if any) - saved separately for scalability
// metadata field is saved separately via saveVerbMetadata()
}
const key = this.getVerbKey(edge.id)
@ -814,10 +824,19 @@ export class R2Storage extends BaseStorage {
connections.set(Number(level), new Set(verbIds as string[]))
}
// ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
const edge: Edge = {
id: data.id,
vector: data.vector,
connections
connections,
// CORE RELATIONAL DATA (read from vector file)
verb: data.verb,
sourceId: data.sourceId,
targetId: data.targetId,
// User metadata (retrieved separately via getVerbMetadata())
metadata: data.metadata
}
this.verbCacheManager.set(id, edge)

20
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -1515,12 +1515,19 @@ export class S3CompatibleStorage extends BaseStorage {
// Convert connections Map to a serializable format
// CRITICAL: Only save lightweight vector data (no metadata)
// Metadata is saved separately via saveVerbMetadata() (2-file system)
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
const serializableEdge = {
id: edge.id,
vector: edge.vector,
connections: this.mapToObject(edge.connections, (set) =>
Array.from(set as Set<string>)
)
),
// CORE RELATIONAL DATA (v3.50.1+)
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
// NO metadata field - saved separately for scalability
}
@ -1640,10 +1647,19 @@ export class S3CompatibleStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
const edge = {
id: parsedEdge.id,
vector: parsedEdge.vector,
connections
connections,
// CORE RELATIONAL DATA (read from vector file)
verb: parsedEdge.verb,
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId,
// User metadata (retrieved separately via getVerbMetadata())
metadata: parsedEdge.metadata
}
this.logger.trace(`Successfully retrieved edge ${id}`)

43
src/storage/adapters/typeAwareStorageAdapter.ts
View file

@ -206,27 +206,23 @@ export class TypeAwareStorageAdapter extends BaseStorage {
}
/**
* Get verb type from verb object or cache
* Get verb type from verb object
*
* ARCHITECTURAL FIX (v3.50.1): Simplified - verb field is now always present
*/
private getVerbType(verb: HNSWVerb | GraphVerb): VerbType {
// Try verb property first
// v3.50.1+: verb is a required field in HNSWVerb
if ('verb' in verb && verb.verb) {
return verb.verb as VerbType
}
// Try type property
// Fallback for GraphVerb (type alias)
if ('type' in verb && verb.type) {
return verb.type as VerbType
}
// Try cache
const cached = this.verbTypeCache.get(verb.id)
if (cached) {
return cached
}
// Default to 'relatedTo' if unknown
console.warn(`[TypeAwareStorage] Unknown verb type for ${verb.id}, defaulting to 'relatedTo'`)
// This should never happen with v3.50.1+ data
console.warn(`[TypeAwareStorage] Verb missing type field for ${verb.id}, defaulting to 'relatedTo'`)
return 'relatedTo'
}
@ -355,9 +351,13 @@ export class TypeAwareStorageAdapter extends BaseStorage {
/**
* Save verb (type-first path)
*
* ARCHITECTURAL FIX (v3.50.1): No more caching hack needed!
* HNSWVerb now includes verb field, so type is always available
*/
protected async saveVerb_internal(verb: HNSWVerb): Promise<void> {
const type = this.getVerbType(verb)
// Type is now a first-class field in HNSWVerb - no caching needed!
const type = verb.verb as VerbType
const path = getVerbVectorPath(type, verb.id)
// Update type tracking
@ -376,16 +376,20 @@ export class TypeAwareStorageAdapter extends BaseStorage {
/**
* Get verb (type-first path)
*
* ARCHITECTURAL FIX (v3.50.1): Cache still useful for performance
* Once we know where a verb is, we can retrieve it O(1) instead of searching all types
*/
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
// Try cache first
// Try cache first for O(1) retrieval
const cachedType = this.verbTypeCache.get(id)
if (cachedType) {
const path = getVerbVectorPath(cachedType, id)
return await this.u.readObjectFromPath(path)
const verb = await this.u.readObjectFromPath(path)
return verb
}
// Search across all types
// Search across all types (only on first access)
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
const type = TypeUtils.getVerbFromIndex(i)
const path = getVerbVectorPath(type, id)
@ -393,7 +397,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
try {
const verb = await this.u.readObjectFromPath(path)
if (verb) {
this.verbTypeCache.set(id, type)
// Cache the type for next time (read from verb.verb field)
this.verbTypeCache.set(id, verb.verb as VerbType)
return verb
}
} catch (error) {
@ -474,6 +479,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
/**
* Get verbs by type (O(1) with type-first paths!)
*
* ARCHITECTURAL FIX (v3.50.1): Type is now in HNSWVerb, cached on read
*/
protected async getVerbsByType_internal(verbType: string): Promise<GraphVerb[]> {
const type = verbType as VerbType
@ -486,11 +493,13 @@ export class TypeAwareStorageAdapter extends BaseStorage {
try {
const hnswVerb = await this.u.readObjectFromPath(path)
if (hnswVerb) {
// Cache type from HNSWVerb for future O(1) retrievals
this.verbTypeCache.set(hnswVerb.id, hnswVerb.verb as VerbType)
// Convert to GraphVerb
const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb)
if (graphVerb) {
verbs.push(graphVerb)
this.verbTypeCache.set(hnswVerb.id, type)
}
}
} catch (error) {