soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: add verb and noun metadata handling in storage adapters

Browse source 
- Implement saveVerbMetadata and getVerbMetadata methods for managing verb metadata.
- Implement saveNounMetadata and getNounMetadata methods for managing noun metadata.
- Update storage adapters to use HNSWVerb instead of GraphVerb for improved performance.
- Deprecate methods that require loading metadata for edges, returning empty arrays instead.
This commit is contained in:
David Snelling 2025-08-03 10:47:55 -07:00
parent 7f8c8c0630
commit 42eccc051e
4 changed files with 386 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

161
src/storage/adapters/opfsStorage.ts
View file

@ -3,17 +3,17 @@
* Provides persistent storage for the vector database using the Origin Private File System API
*/
import {GraphVerb, HNSWNoun, StatisticsData} from '../../coreTypes.js'
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js'
import {GraphVerb, HNSWNoun, HNSWVerb, StatisticsData} from '../../coreTypes.js'
import {BaseStorage, NOUNS_DIR, VERBS_DIR, METADATA_DIR, NOUN_METADATA_DIR, VERB_METADATA_DIR, INDEX_DIR, STATISTICS_KEY} from '../baseStorage.js'
import '../../types/fileSystemTypes.js'
// Type alias for HNSWNode
type HNSWNode = HNSWNoun
/**
* Type alias for GraphVerb to make the code more readable
* Type alias for HNSWVerb to make the code more readable
*/
type Edge = GraphVerb
type Edge = HNSWVerb
/**
* Helper function to safely get a file from a FileSystemHandle
@ -41,6 +41,8 @@ export class OPFSStorage extends BaseStorage {
private nounsDir: FileSystemDirectoryHandle | null = null
private verbsDir: FileSystemDirectoryHandle | null = null
private metadataDir: FileSystemDirectoryHandle | null = null
private nounMetadataDir: FileSystemDirectoryHandle | null = null
private verbMetadataDir: FileSystemDirectoryHandle | null = null
private indexDir: FileSystemDirectoryHandle | null = null
private isAvailable = false
private isPersistentRequested = false
@ -94,6 +96,16 @@ export class OPFSStorage extends BaseStorage {
create: true
})
// Create or get noun metadata directory
this.nounMetadataDir = await this.rootDir.getDirectoryHandle(NOUN_METADATA_DIR, {
create: true
})
// Create or get verb metadata directory
this.verbMetadataDir = await this.rootDir.getDirectoryHandle(VERB_METADATA_DIR, {
create: true
})
// Create or get index directory
this.indexDir = await this.rootDir.getDirectoryHandle(INDEX_DIR, {
create: true
@ -345,7 +357,7 @@ export class OPFSStorage extends BaseStorage {
/**
* Save a verb to storage (internal implementation)
*/
protected async saveVerb_internal(verb: GraphVerb): Promise<void> {
protected async saveVerb_internal(verb: HNSWVerb): Promise<void> {
return this.saveEdge(verb)
}
@ -382,7 +394,7 @@ export class OPFSStorage extends BaseStorage {
/**
* Get a verb from storage (internal implementation)
*/
protected async getVerb_internal(id: string): Promise<GraphVerb | null> {
protected async getVerb_internal(id: string): Promise<HNSWVerb | null> {
return this.getEdge(id)
}
@ -422,17 +434,7 @@ export class OPFSStorage extends BaseStorage {
return {
id: data.id,
vector: data.vector,
connections,
sourceId: data.sourceId || data.source,
targetId: data.targetId || data.target,
source: data.sourceId || data.source,
target: data.targetId || data.target,
verb: data.type || data.verb,
weight: data.weight,
metadata: data.metadata,
createdAt: data.createdAt || defaultTimestamp,
updatedAt: data.updatedAt || defaultTimestamp,
createdBy: data.createdBy || defaultCreatedBy
connections
}
} catch (error) {
// Edge not found or other error
@ -443,7 +445,7 @@ export class OPFSStorage extends BaseStorage {
/**
* Get all verbs from storage (internal implementation)
*/
protected async getAllVerbs_internal(): Promise<GraphVerb[]> {
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
return this.getAllEdges()
}
@ -485,17 +487,7 @@ export class OPFSStorage extends BaseStorage {
allEdges.push({
id: data.id,
vector: data.vector,
connections,
sourceId: data.sourceId || data.source,
targetId: data.targetId || data.target,
source: data.sourceId || data.source,
target: data.targetId || data.target,
verb: data.type || data.verb,
weight: data.weight,
metadata: data.metadata,
createdAt: data.createdAt || defaultTimestamp,
updatedAt: data.updatedAt || defaultTimestamp,
createdBy: data.createdBy || defaultCreatedBy
connections
})
} catch (error) {
console.error(`Error reading edge file ${name}:`, error)
@ -513,45 +505,60 @@ export class OPFSStorage extends BaseStorage {
* Get verbs by source (internal implementation)
*/
protected async getVerbsBySource_internal(sourceId: string): Promise<GraphVerb[]> {
return this.getEdgesBySource(sourceId)
// This method is deprecated and would require loading metadata for each edge
// For now, return empty array since this is not efficiently implementable with new storage pattern
console.warn('getVerbsBySource_internal is deprecated and not efficiently supported in new storage pattern')
return []
}
/**
* Get edges by source
*/
protected async getEdgesBySource(sourceId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => (edge.sourceId || edge.source) === sourceId)
protected async getEdgesBySource(sourceId: string): Promise<GraphVerb[]> {
// This method is deprecated and would require loading metadata for each edge
// For now, return empty array since this is not efficiently implementable with new storage pattern
console.warn('getEdgesBySource is deprecated and not efficiently supported in new storage pattern')
return []
}
/**
* Get verbs by target (internal implementation)
*/
protected async getVerbsByTarget_internal(targetId: string): Promise<GraphVerb[]> {
return this.getEdgesByTarget(targetId)
// This method is deprecated and would require loading metadata for each edge
// For now, return empty array since this is not efficiently implementable with new storage pattern
console.warn('getVerbsByTarget_internal is deprecated and not efficiently supported in new storage pattern')
return []
}
/**
* Get edges by target
*/
protected async getEdgesByTarget(targetId: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => (edge.targetId || edge.target) === targetId)
protected async getEdgesByTarget(targetId: string): Promise<GraphVerb[]> {
// This method is deprecated and would require loading metadata for each edge
// For now, return empty array since this is not efficiently implementable with new storage pattern
console.warn('getEdgesByTarget is deprecated and not efficiently supported in new storage pattern')
return []
}
/**
* Get verbs by type (internal implementation)
*/
protected async getVerbsByType_internal(type: string): Promise<GraphVerb[]> {
return this.getEdgesByType(type)
// This method is deprecated and would require loading metadata for each edge
// For now, return empty array since this is not efficiently implementable with new storage pattern
console.warn('getVerbsByType_internal is deprecated and not efficiently supported in new storage pattern')
return []
}
/**
* Get edges by type
*/
protected async getEdgesByType(type: string): Promise<Edge[]> {
const edges = await this.getAllEdges()
return edges.filter((edge) => (edge.type || edge.verb) === type)
protected async getEdgesByType(type: string): Promise<GraphVerb[]> {
// This method is deprecated and would require loading metadata for each edge
// For now, return empty array since this is not efficiently implementable with new storage pattern
console.warn('getEdgesByType is deprecated and not efficiently supported in new storage pattern')
return []
}
/**
@ -620,6 +627,72 @@ export class OPFSStorage extends BaseStorage {
}
}
/**
* Save verb metadata to storage
*/
public async saveVerbMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
const fileName = `${id}.json`
const fileHandle = await (this.verbMetadataDir as FileSystemDirectoryHandle).getFileHandle(fileName, { create: true })
const writable = await (fileHandle as FileSystemFileHandle).createWritable()
await writable.write(JSON.stringify(metadata, null, 2))
await writable.close()
}
/**
* Get verb metadata from storage
*/
public async getVerbMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
const fileName = `${id}.json`
try {
const fileHandle = await (this.verbMetadataDir as FileSystemDirectoryHandle).getFileHandle(fileName)
const file = await safeGetFile(fileHandle)
const text = await file.text()
return JSON.parse(text)
} catch (error: any) {
if (error.name !== 'NotFoundError') {
console.error(`Error reading verb metadata ${id}:`, error)
}
return null
}
}
/**
* Save noun metadata to storage
*/
public async saveNounMetadata(id: string, metadata: any): Promise<void> {
await this.ensureInitialized()
const fileName = `${id}.json`
const fileHandle = await (this.nounMetadataDir as FileSystemDirectoryHandle).getFileHandle(fileName, { create: true })
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(metadata, null, 2))
await writable.close()
}
/**
* Get noun metadata from storage
*/
public async getNounMetadata(id: string): Promise<any | null> {
await this.ensureInitialized()
const fileName = `${id}.json`
try {
const fileHandle = await (this.nounMetadataDir as FileSystemDirectoryHandle).getFileHandle(fileName)
const file = await safeGetFile(fileHandle)
const text = await file.text()
return JSON.parse(text)
} catch (error: any) {
if (error.name !== 'NotFoundError') {
console.error(`Error reading noun metadata ${id}:`, error)
}
return null
}
}
/**
* Clear all data from storage
*/
@ -651,6 +724,12 @@ export class OPFSStorage extends BaseStorage {
// Remove all files in the metadata directory
await removeDirectoryContents(this.metadataDir!)
// Remove all files in the noun metadata directory
await removeDirectoryContents(this.nounMetadataDir!)
// Remove all files in the verb metadata directory
await removeDirectoryContents(this.verbMetadataDir!)
// Remove all files in the index directory
await removeDirectoryContents(this.indexDir!)
} catch (error) {