soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: v5.1.0 - VFS auto-initialization and complete API documentation

Browse source 
BREAKING CHANGES:
- VFS API changed from brain.vfs() (method) to brain.vfs (property)
- VFS now auto-initializes during brain.init() - no separate vfs.init() needed

Features:
- VFS auto-initialization with property access pattern
- Complete TypeAware COW support verification (all 20 methods)
- Comprehensive API documentation (docs/api/README.md)
- All 7 storage adapters verified with COW support

Bug Fixes:
- CLI now properly initializes brain before VFS operations
- Fixed infinite recursion in VFS initialization
- All VFS CLI commands updated to modern API

Documentation:
- Created comprehensive, verified API reference
- Consolidated documentation structure (deleted redundant quick starts)
- Updated all VFS docs to v5.1.0 patterns
- Fixed all internal documentation links

Verification:
- Memory Storage: 23/24 tests (95.8%)
- FileSystem Storage: 9/9 tests (100%)
- VFS Auto-Init: 7/7 tests (100%)
- Zero fake code confirmed
This commit is contained in:
David Snelling 2025-11-02 10:58:52 -08:00
parent 5e16f9e5e8
commit d4c9f71345
20 changed files with 2306 additions and 1343 deletions
Download patch file Download diff file Expand all files Collapse all files

60
src/storage/adapters/memoryStorage.ts
View file

@ -82,6 +82,7 @@ export class MemoryStorage extends BaseStorage {
/**
* Save a noun to storage (v4.0.0: pure vector only, no metadata)
* v5.0.1: COW-aware - uses branch-prefixed paths for fork isolation
*/
protected async saveNoun_internal(noun: HNSWNoun): Promise<void> {
const isNew = !this.nouns.has(noun.id)
@ -102,7 +103,13 @@ export class MemoryStorage extends BaseStorage {
nounCopy.connections.set(level, new Set(connections))
}
// Save the noun directly in the nouns map
// v5.0.1: COW-aware write using branch-prefixed path
// Use synthetic path for vector storage (nouns don't have types in standalone mode)
const path = `hnsw/nouns/${noun.id}.json`
await this.writeObjectToBranch(path, nounCopy)
// ALSO store in nouns Map for fast iteration (getNouns, initializeCounts)
// This is redundant but maintains backward compatibility
this.nouns.set(noun.id, nounCopy)
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
@ -112,10 +119,12 @@ export class MemoryStorage extends BaseStorage {
/**
* Get a noun from storage (v4.0.0: returns pure vector only)
* Base class handles combining with metadata
* v5.0.1: COW-aware - reads from branch-prefixed paths with inheritance
*/
protected async getNoun_internal(id: string): Promise<HNSWNoun | null> {
// Get the noun directly from the nouns map
const noun = this.nouns.get(id)
// v5.0.1: COW-aware read using branch-prefixed path with inheritance
const path = `hnsw/nouns/${id}.json`
const noun = await this.readWithInheritance(path)
// If not found, return null
if (!noun) {
@ -132,9 +141,10 @@ export class MemoryStorage extends BaseStorage {
// ✅ NO metadata field in v4.0.0
}
// Copy connections
for (const [level, connections] of noun.connections.entries()) {
nounCopy.connections.set(level, new Set(connections))
// Copy connections (handle both Map and plain object from JSON)
const connections = noun.connections instanceof Map ? noun.connections : new Map(Object.entries(noun.connections || {}))
for (const [level, conns] of connections.entries()) {
nounCopy.connections.set(Number(level), new Set(conns))
}
return nounCopy
@ -320,6 +330,7 @@ export class MemoryStorage extends BaseStorage {
/**
* Delete a noun from storage (v4.0.0)
* v5.0.1: COW-aware - deletes from branch-prefixed paths
*/
protected async deleteNoun_internal(id: string): Promise<void> {
// v4.0.0: Get type from separate metadata storage
@ -328,11 +339,18 @@ export class MemoryStorage extends BaseStorage {
const type = metadata.noun || 'default'
this.decrementEntityCount(type)
}
// v5.0.1: COW-aware delete using branch-prefixed path
const path = `hnsw/nouns/${id}.json`
await this.deleteObjectFromBranch(path)
// Also remove from nouns Map for fast iteration
this.nouns.delete(id)
}
/**
* Save a verb to storage (v4.0.0: pure vector + core fields, no metadata)
* v5.0.1: COW-aware - uses branch-prefixed paths for fork isolation
*/
protected async saveVerb_internal(verb: HNSWVerb): Promise<void> {
const isNew = !this.verbs.has(verb.id)
@ -356,7 +374,11 @@ export class MemoryStorage extends BaseStorage {
verbCopy.connections.set(level, new Set(connections))
}
// Save the verb directly in the verbs map
// v5.0.1: COW-aware write using branch-prefixed path
const path = `hnsw/verbs/${verb.id}.json`
await this.writeObjectToBranch(path, verbCopy)
// ALSO store in verbs Map for fast iteration (getVerbs, initializeCounts)
this.verbs.set(verb.id, verbCopy)
// Note: Count tracking happens in saveVerbMetadata since metadata is separate
@ -365,10 +387,12 @@ export class MemoryStorage extends BaseStorage {
/**
* Get a verb from storage (v4.0.0: returns pure vector + core fields)
* Base class handles combining with metadata
* v5.0.1: COW-aware - reads from branch-prefixed paths with inheritance
*/
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
// Get the verb directly from the verbs map
const verb = this.verbs.get(id)
// v5.0.1: COW-aware read using branch-prefixed path with inheritance
const path = `hnsw/verbs/${id}.json`
const verb = await this.readWithInheritance(path)
// If not found, return null
if (!verb) {
@ -389,9 +413,10 @@ export class MemoryStorage extends BaseStorage {
// ✅ NO metadata field in v4.0.0
}
// Copy connections
for (const [level, connections] of verb.connections.entries()) {
verbCopy.connections.set(level, new Set(connections))
// Copy connections (handle both Map and plain object from JSON)
const connections = verb.connections instanceof Map ? verb.connections : new Map(Object.entries(verb.connections || {}))
for (const [level, conns] of connections.entries()) {
verbCopy.connections.set(Number(level), new Set(conns))
}
return verbCopy
@ -595,11 +620,9 @@ export class MemoryStorage extends BaseStorage {
/**
* Delete a verb from storage
* v5.0.1: COW-aware - deletes from branch-prefixed paths
*/
protected async deleteVerb_internal(id: string): Promise<void> {
// Delete the HNSWVerb from the verbs map
this.verbs.delete(id)
// CRITICAL: Also delete verb metadata - this is what getVerbs() uses to find verbs
// Without this, getVerbsBySource() will still find "deleted" verbs via their metadata
const metadata = await this.getVerbMetadata(id)
@ -610,6 +633,13 @@ export class MemoryStorage extends BaseStorage {
// Delete the metadata using the base storage method
await this.deleteVerbMetadata(id)
}
// v5.0.1: COW-aware delete using branch-prefixed path
const path = `hnsw/verbs/${id}.json`
await this.deleteObjectFromBranch(path)
// Also remove from verbs Map for fast iteration
this.verbs.delete(id)
}
/**

354
src/storage/adapters/typeAwareStorageAdapter.ts
View file

@ -41,6 +41,7 @@ import {
GraphVerb,
HNSWNoun,
HNSWVerb,
HNSWNounWithMetadata,
HNSWVerbWithMetadata,
NounMetadata,
VerbMetadata,
@ -262,8 +263,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
this.nounCountsByType[typeIndex]++
this.nounTypeCache.set(noun.id, type)
// Delegate to underlying storage
await this.u.writeObjectToPath(path, noun)
// COW-aware write (v5.0.1): Use COW helper for branch isolation
await this.writeObjectToBranch(path, noun)
// Periodically save statistics (every 100 saves)
if (this.nounCountsByType[typeIndex] % 100 === 0) {
@ -279,7 +280,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const cachedType = this.nounTypeCache.get(id)
if (cachedType) {
const path = getNounVectorPath(cachedType, id)
return await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
return await this.readWithInheritance(path)
}
// Need to search across all types (expensive, but cached after first access)
@ -288,7 +290,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const path = getNounVectorPath(type, id)
try {
const noun = await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
const noun = await this.readWithInheritance(path)
if (noun) {
// Cache the type for next time
this.nounTypeCache.set(id, type)
@ -309,14 +312,15 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const type = nounType as NounType
const prefix = `entities/nouns/${type}/vectors/`
// List all files under this type's directory
const paths = await this.u.listObjectsUnderPath(prefix)
// COW-aware list (v5.0.1): Use COW helper for branch isolation
const paths = await this.listObjectsInBranch(prefix)
// Load all nouns of this type
const nouns: HNSWNoun[] = []
for (const path of paths) {
try {
const noun = await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
const noun = await this.readWithInheritance(path)
if (noun) {
nouns.push(noun)
// Cache the type
@ -338,7 +342,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const cachedType = this.nounTypeCache.get(id)
if (cachedType) {
const path = getNounVectorPath(cachedType, id)
await this.u.deleteObjectFromPath(path)
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
await this.deleteObjectFromBranch(path)
// Update counts
const typeIndex = TypeUtils.getNounIndex(cachedType)
@ -355,7 +360,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const path = getNounVectorPath(type, id)
try {
await this.u.deleteObjectFromPath(path)
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
await this.deleteObjectFromBranch(path)
// Update counts
if (this.nounCountsByType[i] > 0) {
@ -385,8 +391,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
this.verbCountsByType[typeIndex]++
this.verbTypeCache.set(verb.id, type)
// Delegate to underlying storage
await this.u.writeObjectToPath(path, verb)
// COW-aware write (v5.0.1): Use COW helper for branch isolation
await this.writeObjectToBranch(path, verb)
// Periodically save statistics
if (this.verbCountsByType[typeIndex] % 100 === 0) {
@ -405,7 +411,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const cachedType = this.verbTypeCache.get(id)
if (cachedType) {
const path = getVerbVectorPath(cachedType, id)
const verb = await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
const verb = await this.readWithInheritance(path)
return verb
}
@ -415,7 +422,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const path = getVerbVectorPath(type, id)
try {
const verb = await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
const verb = await this.readWithInheritance(path)
if (verb) {
// Cache the type for next time (read from verb.verb field)
this.verbTypeCache.set(id, verb.verb as VerbType)
@ -433,29 +441,39 @@ export class TypeAwareStorageAdapter extends BaseStorage {
* Get verbs by source
*/
protected async getVerbsBySource_internal(sourceId: string): Promise<HNSWVerbWithMetadata[]> {
// v4.8.1 PERFORMANCE FIX: Delegate to underlying storage instead of scanning all files
// Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files
// This was the root cause of the 11-version VFS bug (timeouts/zero results)
// v5.0.1 COW FIX: Use getVerbsWithPagination which is COW-aware
// Previous v4.8.1 implementation delegated to underlying storage, which bypasses COW!
// The underlying storage delegates to GraphAdjacencyIndex, which is shared between forks.
// This caused getRelations() to return 0 results for fork-created relationships.
//
// Underlying storage adapters have optimized implementations:
// - FileSystemStorage: Uses getVerbsWithPagination with sourceId filter
// - GcsStorage: Uses batch queries with prefix filtering
// - S3Storage: Uses listObjects with sourceId-based filtering
// Now we use getVerbsWithPagination with sourceId filter, which:
// - Searches across all verb types using COW-aware listObjectsInBranch()
// - Reads verbs using COW-aware readWithInheritance()
// - Properly isolates fork data from parent
//
// Phase 1b TODO: Add graph adjacency index query for O(1) lookups:
// const verbIds = await this.graphIndex?.getOutgoingEdges(sourceId) || []
// return Promise.all(verbIds.map(id => this.getVerb(id)))
// Performance: Still efficient because sourceId filter reduces iteration
const result = await this.getVerbsWithPagination({
limit: 10000, // High limit to get all verbs for this source
offset: 0,
filter: { sourceId }
})
return this.underlying.getVerbsBySource(sourceId)
return result.items
}
/**
* Get verbs by target
*/
protected async getVerbsByTarget_internal(targetId: string): Promise<HNSWVerbWithMetadata[]> {
// v4.8.1 PERFORMANCE FIX: Delegate to underlying storage (same as getVerbsBySource fix)
// Previous implementation was O(total_verbs) - scanned ALL 40 verb types and ALL verb files
return this.underlying.getVerbsByTarget(targetId)
// v5.0.1 COW FIX: Use getVerbsWithPagination which is COW-aware
// Same fix as getVerbsBySource_internal - delegating to underlying bypasses COW
const result = await this.getVerbsWithPagination({
limit: 10000, // High limit to get all verbs for this target
offset: 0,
filter: { targetId }
})
return result.items
}
/**
@ -467,12 +485,14 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const type = verbType as VerbType
const prefix = `entities/verbs/${type}/vectors/`
const paths = await this.u.listObjectsUnderPath(prefix)
// COW-aware list (v5.0.1): Use COW helper for branch isolation
const paths = await this.listObjectsInBranch(prefix)
const verbs: HNSWVerbWithMetadata[] = []
for (const path of paths) {
try {
const hnswVerb = await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
const hnswVerb = await this.readWithInheritance(path)
if (!hnswVerb) continue
// Cache type from HNSWVerb for future O(1) retrievals
@ -529,7 +549,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const cachedType = this.verbTypeCache.get(id)
if (cachedType) {
const path = getVerbVectorPath(cachedType, id)
await this.u.deleteObjectFromPath(path)
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
await this.deleteObjectFromBranch(path)
const typeIndex = TypeUtils.getVerbIndex(cachedType)
if (this.verbCountsByType[typeIndex] > 0) {
@ -545,7 +566,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const path = getVerbVectorPath(type, id)
try {
await this.u.deleteObjectFromPath(path)
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
await this.deleteObjectFromBranch(path)
if (this.verbCountsByType[i] > 0) {
this.verbCountsByType[i]--
@ -568,9 +590,9 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const type = (metadata.noun || 'thing') as NounType
this.nounTypeCache.set(id, type)
// Save to type-aware path
// COW-aware write (v5.0.1): Use COW helper for branch isolation
const path = getNounMetadataPath(type, id)
await this.u.writeObjectToPath(path, metadata)
await this.writeObjectToBranch(path, metadata)
}
/**
@ -581,7 +603,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const cachedType = this.nounTypeCache.get(id)
if (cachedType) {
const path = getNounMetadataPath(cachedType, id)
return await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
return await this.readWithInheritance(path)
}
// Search across all types
@ -590,7 +613,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const path = getNounMetadataPath(type, id)
try {
const metadata = await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
const metadata = await this.readWithInheritance(path)
if (metadata) {
// Cache the type for next time
const metadataType = (metadata.noun || 'thing') as NounType
@ -612,7 +636,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const cachedType = this.nounTypeCache.get(id)
if (cachedType) {
const path = getNounMetadataPath(cachedType, id)
await this.u.deleteObjectFromPath(path)
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
await this.deleteObjectFromBranch(path)
return
}
@ -622,7 +647,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const path = getNounMetadataPath(type, id)
try {
await this.u.deleteObjectFromPath(path)
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
await this.deleteObjectFromBranch(path)
return
} catch (error) {
// Not in this type, continue
@ -653,7 +679,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
// Save to type-aware path
const path = getVerbMetadataPath(type, id)
await this.u.writeObjectToPath(path, metadata)
// COW-aware write (v5.0.1): Use COW helper for branch isolation
await this.writeObjectToBranch(path, metadata)
}
/**
@ -664,7 +691,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const cachedType = this.verbTypeCache.get(id)
if (cachedType) {
const path = getVerbMetadataPath(cachedType, id)
return await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
return await this.readWithInheritance(path)
}
// Search across all types
@ -673,7 +701,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const path = getVerbMetadataPath(type, id)
try {
const metadata = await this.u.readObjectFromPath(path)
// COW-aware read (v5.0.1): Use COW helper for branch isolation
const metadata = await this.readWithInheritance(path)
if (metadata) {
// Cache the type for next time
this.verbTypeCache.set(id, type)
@ -694,7 +723,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const cachedType = this.verbTypeCache.get(id)
if (cachedType) {
const path = getVerbMetadataPath(cachedType, id)
await this.u.deleteObjectFromPath(path)
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
await this.deleteObjectFromBranch(path)
return
}
@ -704,7 +734,8 @@ export class TypeAwareStorageAdapter extends BaseStorage {
const path = getVerbMetadataPath(type, id)
try {
await this.u.deleteObjectFromPath(path)
// COW-aware delete (v5.0.1): Use COW helper for branch isolation
await this.deleteObjectFromBranch(path)
return
} catch (error) {
// Not in this type, continue
@ -885,6 +916,245 @@ export class TypeAwareStorageAdapter extends BaseStorage {
return null
}
/**
* Get nouns with pagination (v5.0.1: COW-aware)
* Required for find() to work with TypeAwareStorage
*/
async getNounsWithPagination(options: {
limit?: number
offset?: number
cursor?: string
filter?: any
}): Promise<{
items: HNSWNounWithMetadata[]
totalCount: number
hasMore: boolean
nextCursor?: string
}> {
const limit = options.limit || 100
const offset = options.offset || 0
const filter = options.filter || {}
// Determine which types to search
let typesToSearch: NounType[]
if (filter.nounType) {
typesToSearch = Array.isArray(filter.nounType) ? filter.nounType : [filter.nounType]
} else {
// Search all 31 types
typesToSearch = []
for (let i = 0; i < NOUN_TYPE_COUNT; i++) {
const type = TypeUtils.getNounFromIndex(i)
typesToSearch.push(type)
}
}
// Collect all matching nouns across types (COW-aware!)
const allNouns: HNSWNounWithMetadata[] = []
for (const type of typesToSearch) {
const prefix = `entities/nouns/${type}/vectors/`
// COW-aware list with inheritance (v5.0.1): Fork sees parent's nouns too!
const paths = await this.listObjectsWithInheritance(prefix)
for (const path of paths) {
try {
// COW-aware read with inheritance
const noun = await this.readWithInheritance(path)
if (!noun) continue
// Get metadata separately
const metadata = await this.getNounMetadata(noun.id)
if (!metadata) continue
// Filter by service if specified
if (filter.service && metadata.service !== filter.service) continue
// Filter by custom metadata if specified
if (filter.metadata) {
let matches = true
for (const [key, value] of Object.entries(filter.metadata)) {
if ((metadata as any)[key] !== value) {
matches = false
break
}
}
if (!matches) continue
}
// Extract standard fields from metadata
const { noun: nounType, createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadata as any
// Create HNSWNounWithMetadata (v4.8.0 format)
const nounWithMetadata: HNSWNounWithMetadata = {
id: noun.id,
vector: noun.vector,
connections: noun.connections,
level: noun.level || 0,
type: (nounType as NounType) || NounType.Thing,
createdAt: (createdAt as number) || Date.now(),
updatedAt: (updatedAt as number) || Date.now(),
confidence,
weight,
service,
data,
createdBy,
metadata: customMetadata
}
allNouns.push(nounWithMetadata)
} catch (error) {
// Skip entities with errors
continue
}
}
}
// Apply pagination
const totalCount = allNouns.length
const paginatedNouns = allNouns.slice(offset, offset + limit)
const hasMore = offset + limit < totalCount
// Generate cursor if more results exist
let nextCursor: string | undefined
if (hasMore && paginatedNouns.length > 0) {
nextCursor = paginatedNouns[paginatedNouns.length - 1].id
}
return {
items: paginatedNouns,
totalCount,
hasMore,
nextCursor
}
}
/**
* Get verbs with pagination (v5.0.1: COW-aware)
* Required for GraphAdjacencyIndex rebuild and find() to work
*/
async getVerbsWithPagination(options: {
limit?: number
offset?: number
cursor?: string
filter?: any
}): Promise<{
items: HNSWVerbWithMetadata[]
totalCount: number
hasMore: boolean
nextCursor?: string
}> {
const limit = options.limit || 100
const offset = options.offset || 0
const filter = options.filter || {}
// Determine which types to search
let typesToSearch: VerbType[]
if (filter.verbType) {
typesToSearch = Array.isArray(filter.verbType) ? filter.verbType : [filter.verbType]
} else {
// Search all 40 verb types
typesToSearch = []
for (let i = 0; i < VERB_TYPE_COUNT; i++) {
const type = TypeUtils.getVerbFromIndex(i)
typesToSearch.push(type)
}
}
// Collect all matching verbs across types (COW-aware!)
const allVerbs: HNSWVerbWithMetadata[] = []
for (const type of typesToSearch) {
const prefix = `entities/verbs/${type}/vectors/`
// COW-aware list with inheritance (v5.0.1): Fork sees parent's verbs too!
const paths = await this.listObjectsWithInheritance(prefix)
for (const path of paths) {
try {
// COW-aware read with inheritance
const verb = await this.readWithInheritance(path)
if (!verb) continue
// Filter by sourceId if specified
if (filter.sourceId) {
const sourceIds = Array.isArray(filter.sourceId) ? filter.sourceId : [filter.sourceId]
if (!sourceIds.includes(verb.sourceId)) continue
}
// Filter by targetId if specified
if (filter.targetId) {
const targetIds = Array.isArray(filter.targetId) ? filter.targetId : [filter.targetId]
if (!targetIds.includes(verb.targetId)) continue
}
// Get metadata separately
const metadata = await this.getVerbMetadata(verb.id)
// Filter by service if specified
if (filter.service && metadata && metadata.service !== filter.service) continue
// Filter by custom metadata if specified
if (filter.metadata && metadata) {
let matches = true
for (const [key, value] of Object.entries(filter.metadata)) {
if ((metadata as any)[key] !== value) {
matches = false
break
}
}
if (!matches) continue
}
// Extract standard fields from metadata
const metadataObj = metadata || {}
const { createdAt, updatedAt, confidence, weight, service, data, createdBy, ...customMetadata } = metadataObj as any
// Create HNSWVerbWithMetadata (v4.8.0 format)
const verbWithMetadata: HNSWVerbWithMetadata = {
id: verb.id,
vector: verb.vector,
connections: verb.connections,
verb: verb.verb,
sourceId: verb.sourceId,
targetId: verb.targetId,
createdAt: (createdAt as number) || Date.now(),
updatedAt: (updatedAt as number) || Date.now(),
confidence,
weight,
service,
data,
createdBy,
metadata: customMetadata
}
allVerbs.push(verbWithMetadata)
} catch (error) {
// Skip verbs with errors
continue
}
}
}
// Apply pagination
const totalCount = allVerbs.length
const paginatedVerbs = allVerbs.slice(offset, offset + limit)
const hasMore = offset + limit < totalCount
// Generate cursor if more results exist
let nextCursor: string | undefined
if (hasMore && paginatedVerbs.length > 0) {
nextCursor = paginatedVerbs[paginatedVerbs.length - 1].id
}
return {
items: paginatedVerbs,
totalCount,
hasMore,
nextCursor
}
}
/**
* Save HNSW system data (entry point, max level)
*/

182
src/storage/baseStorage.ts
View file

@ -196,6 +196,21 @@ export abstract class BaseStorage extends BaseStorageAdapter {
}
}
/**
* Lightweight COW enablement - just enables branch-scoped paths
* Called during init() to ensure all data is stored with branch prefixes from the start
* RefManager/BlobStorage/CommitLog are lazy-initialized on first fork()
* @param branch - Branch name to use (default: 'main')
*/
public enableCOWLightweight(branch: string = 'main'): void {
if (this.cowEnabled) {
return
}
this.currentBranch = branch
this.cowEnabled = true
// RefManager/BlobStorage/CommitLog remain undefined until first fork()
}
/**
* Initialize COW (Copy-on-Write) support
* Creates RefManager and BlobStorage for instant fork() capability
@ -211,13 +226,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
branch?: string
enableCompression?: boolean
}): Promise<void> {
if (this.cowEnabled) {
// Already initialized
// Check if RefManager already initialized (full COW setup complete)
if (this.refManager) {
return
}
// Set current branch
this.currentBranch = options?.branch || 'main'
// Enable lightweight COW if not already enabled
if (!this.cowEnabled) {
this.currentBranch = options?.branch || 'main'
this.cowEnabled = true
}
// Create COWStorageAdapter bridge
// This adapts BaseStorage's methods to the simple key-value interface
@ -311,6 +329,138 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.cowEnabled = true
}
/**
* Resolve branch-scoped path for COW isolation
* @protected - Available to subclasses for COW implementation
*/
protected resolveBranchPath(basePath: string, branch?: string): string {
if (!this.cowEnabled) {
return basePath // COW disabled, use direct path
}
const targetBranch = branch || this.currentBranch || 'main'
// Branch-scoped path: branches/<branch>/<basePath>
return `branches/${targetBranch}/${basePath}`
}
/**
* Write object to branch-specific path (COW layer)
* @protected - Available to subclasses for COW implementation
*/
protected async writeObjectToBranch(path: string, data: any, branch?: string): Promise<void> {
const branchPath = this.resolveBranchPath(path, branch)
return this.writeObjectToPath(branchPath, data)
}
/**
* Read object with inheritance from parent branches (COW layer)
* Tries current branch first, then walks commit history
* @protected - Available to subclasses for COW implementation
*/
protected async readWithInheritance(path: string, branch?: string): Promise<any | null> {
if (!this.cowEnabled) {
// COW disabled, direct read
return this.readObjectFromPath(path)
}
const targetBranch = branch || this.currentBranch || 'main'
// Try current branch first
const branchPath = this.resolveBranchPath(path, targetBranch)
let data = await this.readObjectFromPath(branchPath)
if (data !== null) {
return data // Found in current branch
}
// Not in branch, check if we're on main (no inheritance needed)
if (targetBranch === 'main') {
return null
}
// Not in branch, walk commit history to find in parent
if (this.refManager && this.commitLog) {
try {
const commitHash = await this.refManager.resolveRef(targetBranch)
if (commitHash) {
// Walk parent commits until we find the data
for await (const commit of this.commitLog.walk(commitHash)) {
// Try reading from parent's branch path
const parentBranch = commit.metadata?.branch || 'main'
if (parentBranch === targetBranch) continue // Skip self
const parentPath = this.resolveBranchPath(path, parentBranch)
data = await this.readObjectFromPath(parentPath)
if (data !== null) {
return data // Found in ancestor
}
}
}
} catch (error) {
// Commit walk failed, fall back to main
const mainPath = this.resolveBranchPath(path, 'main')
return this.readObjectFromPath(mainPath)
}
}
// Last fallback: try main branch
const mainPath = this.resolveBranchPath(path, 'main')
return this.readObjectFromPath(mainPath)
}
/**
* Delete object from branch-specific path (COW layer)
* @protected - Available to subclasses for COW implementation
*/
protected async deleteObjectFromBranch(path: string, branch?: string): Promise<void> {
const branchPath = this.resolveBranchPath(path, branch)
return this.deleteObjectFromPath(branchPath)
}
/**
* List objects under path in branch (COW layer)
* @protected - Available to subclasses for COW implementation
*/
protected async listObjectsInBranch(prefix: string, branch?: string): Promise<string[]> {
const branchPrefix = this.resolveBranchPath(prefix, branch)
const paths = await this.listObjectsUnderPath(branchPrefix)
// Remove branch prefix from results
const targetBranch = branch || this.currentBranch || 'main'
const prefixToRemove = `branches/${targetBranch}/`
return paths.map(p => p.startsWith(prefixToRemove) ? p.substring(prefixToRemove.length) : p)
}
/**
* List objects with inheritance (v5.0.1)
* Lists objects from current branch AND main branch, returns unique paths
* This enables fork to see parent's data in pagination operations
*
* Simplified approach: All branches inherit from main
*/
protected async listObjectsWithInheritance(prefix: string, branch?: string): Promise<string[]> {
if (!this.cowEnabled) {
return this.listObjectsInBranch(prefix, branch)
}
const targetBranch = branch || this.currentBranch || 'main'
// Collect paths from current branch
const pathsSet = new Set<string>()
const currentBranchPaths = await this.listObjectsInBranch(prefix, targetBranch)
currentBranchPaths.forEach(p => pathsSet.add(p))
// If not on main, also list from main (all branches inherit from main)
if (targetBranch !== 'main') {
const mainPaths = await this.listObjectsInBranch(prefix, 'main')
mainPaths.forEach(p => pathsSet.add(p))
}
return Array.from(pathsSet)
}
/**
* Save a noun to storage (v4.0.0: vector only, metadata saved separately)
* @param noun Pure HNSW vector data (no metadata)
@ -1113,7 +1263,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public async saveMetadata(id: string, metadata: NounMetadata): Promise<void> {
await this.ensureInitialized()
const keyInfo = this.analyzeKey(id, 'system')
return this.writeObjectToPath(keyInfo.fullPath, metadata)
return this.writeObjectToBranch(keyInfo.fullPath, metadata)
}
/**
@ -1123,7 +1273,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public async getMetadata(id: string): Promise<NounMetadata | null> {
await this.ensureInitialized()
const keyInfo = this.analyzeKey(id, 'system')
return this.readObjectFromPath(keyInfo.fullPath)
return this.readWithInheritance(keyInfo.fullPath)
}
/**
@ -1151,11 +1301,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Determine if this is a new entity by checking if metadata already exists
const keyInfo = this.analyzeKey(id, 'noun-metadata')
const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath)
const existingMetadata = await this.readWithInheritance(keyInfo.fullPath)
const isNew = !existingMetadata
// Save the metadata
await this.writeObjectToPath(keyInfo.fullPath, metadata)
// Save the metadata (COW-aware - writes to branch-specific path)
await this.writeObjectToBranch(keyInfo.fullPath, metadata)
// CRITICAL FIX (v4.1.2): Increment count for new entities
// This runs AFTER metadata is saved, guaranteeing type information is available
@ -1177,7 +1327,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public async getNounMetadata(id: string): Promise<NounMetadata | null> {
await this.ensureInitialized()
const keyInfo = this.analyzeKey(id, 'noun-metadata')
return this.readObjectFromPath(keyInfo.fullPath)
return this.readWithInheritance(keyInfo.fullPath)
}
/**
@ -1187,7 +1337,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public async deleteNounMetadata(id: string): Promise<void> {
await this.ensureInitialized()
const keyInfo = this.analyzeKey(id, 'noun-metadata')
return this.deleteObjectFromPath(keyInfo.fullPath)
return this.deleteObjectFromBranch(keyInfo.fullPath)
}
/**
@ -1216,11 +1366,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Determine if this is a new verb by checking if metadata already exists
const keyInfo = this.analyzeKey(id, 'verb-metadata')
const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath)
const existingMetadata = await this.readWithInheritance(keyInfo.fullPath)
const isNew = !existingMetadata
// Save the metadata
await this.writeObjectToPath(keyInfo.fullPath, metadata)
// Save the metadata (COW-aware - writes to branch-specific path)
await this.writeObjectToBranch(keyInfo.fullPath, metadata)
// CRITICAL FIX (v4.1.2): Increment verb count for new relationships
// This runs AFTER metadata is saved
@ -1243,7 +1393,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public async getVerbMetadata(id: string): Promise<VerbMetadata | null> {
await this.ensureInitialized()
const keyInfo = this.analyzeKey(id, 'verb-metadata')
return this.readObjectFromPath(keyInfo.fullPath)
return this.readWithInheritance(keyInfo.fullPath)
}
/**
@ -1253,7 +1403,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
public async deleteVerbMetadata(id: string): Promise<void> {
await this.ensureInitialized()
const keyInfo = this.analyzeKey(id, 'verb-metadata')
return this.deleteObjectFromPath(keyInfo.fullPath)
return this.deleteObjectFromBranch(keyInfo.fullPath)
}
/**