soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: introduce CLI for Brainy and enhance type validation

Browse source 
Added a comprehensive command-line interface (CLI) for interacting with the Brainy vector database. The CLI supports various operations, including database initialization, adding/searching nouns, managing relationships, and querying database status. Enhanced type validation logic for nouns and verbs to ensure consistency and enforce default types for invalid inputs. Updated test scripts to verify type validation and edge cases.
This commit is contained in:
David Snelling 2025-06-05 11:18:20 -07:00
parent a0ce5b0ca9
commit cdbd2a9db4
14 changed files with 1196 additions and 373 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/augmentations/memoryAugmentations.ts
View file

@ -176,7 +176,7 @@ abstract class BaseMemoryAugmentation implements IMemoryAugmentation {
}
// Get all nodes from storage
const nodes = await this.storage.getAllNodes()
const nodes = await this.storage.getAllNouns()
// Calculate distances and prepare results
const results: Array<{

258
src/brainyData.ts
View file

@ -8,15 +8,16 @@ import { HNSWIndex } from './hnsw/hnswIndex.js'
import { createStorage } from './storage/opfsStorage.js'
import {
DistanceFunction,
Edge,
GraphVerb,
EmbeddingFunction,
HNSWConfig, HNSWNode,
HNSWConfig, HNSWNoun,
SearchResult,
StorageAdapter,
Vector,
VectorDocument
} from './coreTypes.js'
import { cosineDistance, defaultEmbeddingFunction, euclideanDistance } from './utils/index.js'
import { NounType, VerbType, GraphNoun } from './types/graphTypes.js'
export interface BrainyDataConfig {
/**
@ -160,16 +161,16 @@ export class BrainyData<T = any> {
// Initialize storage
await this.storage!.init()
// Load all nodes from storage
const nodes: HNSWNode[] = await this.storage!.getAllNodes()
// Load all nouns from storage
const nouns: HNSWNoun[] = await this.storage!.getAllNouns()
// Clear the index and add all nodes
// Clear the index and add all nouns
this.index.clear()
for (const node of nodes) {
for (const noun of nouns) {
// Add to index
this.index.addItem({
id: node.id,
vector: node.vector
id: noun.id,
vector: noun.vector
})
}
@ -231,18 +232,32 @@ export class BrainyData<T = any> {
// Add to index
this.index.addItem({ id, vector })
// Get the node from the index
const node = this.index.getNodes().get(id)
// Get the noun from the index
const noun = this.index.getNodes().get(id)
if (!node) {
throw new Error(`Failed to retrieve newly created node with ID ${id}`)
if (!noun) {
throw new Error(`Failed to retrieve newly created noun with ID ${id}`)
}
// Save node to storage
await this.storage!.saveNode(node)
// Save noun to storage
await this.storage!.saveNoun(noun)
// Save metadata if provided
if (metadata !== undefined) {
// Validate noun type if metadata is for a GraphNoun
if (metadata && typeof metadata === 'object' && 'noun' in metadata) {
const nounType = (metadata as any).noun;
// Check if the noun type is valid
const isValidNounType = Object.values(NounType).includes(nounType);
if (!isValidNounType) {
console.warn(`Invalid noun type: ${nounType}. Falling back to GraphNoun.`);
// Set a default noun type
(metadata as any).noun = NounType.Concept;
}
}
await this.storage!.saveMetadata(id, metadata)
}
@ -331,7 +346,7 @@ export class BrainyData<T = any> {
throw new Error('Query vector is undefined or null')
}
// If no noun types specified, search all nodes
// If no noun types specified, search all nouns
if (!nounTypes || nounTypes.length === 0) {
// Search in the index
const results = this.index.search(queryVector, k)
@ -340,8 +355,8 @@ export class BrainyData<T = any> {
const searchResults: SearchResult<T>[] = []
for (const [id, score] of results) {
const node = this.index.getNodes().get(id)
if (!node) {
const noun = this.index.getNodes().get(id)
if (!noun) {
continue
}
@ -350,28 +365,28 @@ export class BrainyData<T = any> {
searchResults.push({
id,
score,
vector: node.vector,
vector: noun.vector,
metadata
})
}
return searchResults
} else {
// Get nodes for each noun type in parallel
const nodePromises = nounTypes.map(nounType => this.storage!.getNodesByNounType(nounType))
const nodeArrays = await Promise.all(nodePromises)
// Get nouns for each noun type in parallel
const nounPromises = nounTypes.map(nounType => this.storage!.getNounsByNounType(nounType))
const nounArrays = await Promise.all(nounPromises)
// Combine all nodes
const nodes: HNSWNode[] = []
for (const nodeArray of nodeArrays) {
nodes.push(...nodeArray)
// Combine all nouns
const nouns: HNSWNoun[] = []
for (const nounArray of nounArrays) {
nouns.push(...nounArray)
}
// Calculate distances for each node
// Calculate distances for each noun
const results: Array<[string, number]> = []
for (const node of nodes) {
const distance = this.index.getDistanceFunction()(queryVector, node.vector)
results.push([node.id, distance])
for (const noun of nouns) {
const distance = this.index.getDistanceFunction()(queryVector, noun.vector)
results.push([noun.id, distance])
}
// Sort by distance (ascending)
@ -384,8 +399,8 @@ export class BrainyData<T = any> {
const searchResults: SearchResult<T>[] = []
for (const [id, score] of topResults) {
const node = nodes.find(n => n.id === id)
if (!node) {
const noun = nouns.find(n => n.id === id)
if (!noun) {
continue
}
@ -394,7 +409,7 @@ export class BrainyData<T = any> {
searchResults.push({
id,
score,
vector: node.vector,
vector: noun.vector,
metadata
})
}
@ -447,22 +462,22 @@ export class BrainyData<T = any> {
if (options.includeVerbs && this.storage) {
for (const result of searchResults) {
try {
// Get outgoing edges (verbs) for this noun
const outgoingEdges = await this.storage.getEdgesBySource(result.id);
// Get outgoing verbs for this noun
const outgoingVerbs = await this.storage.getVerbsBySource(result.id);
// Get incoming edges (verbs) for this noun
const incomingEdges = await this.storage.getEdgesByTarget(result.id);
// Get incoming verbs for this noun
const incomingVerbs = await this.storage.getVerbsByTarget(result.id);
// Combine all edges
const allEdges = [...outgoingEdges, ...incomingEdges];
// Combine all verbs
const allVerbs = [...outgoingVerbs, ...incomingVerbs];
// Add edges to the result metadata
// Add verbs to the result metadata
if (!result.metadata) {
result.metadata = {} as T;
}
// Add the edges to the metadata
(result.metadata as any).associatedVerbs = allEdges;
// Add the verbs to the metadata
(result.metadata as any).associatedVerbs = allVerbs;
} catch (error) {
console.warn(`Failed to retrieve verbs for noun ${result.id}:`, error);
}
@ -479,9 +494,9 @@ export class BrainyData<T = any> {
await this.ensureInitialized()
try {
// Get node from index
const node = this.index.getNodes().get(id)
if (!node) {
// Get noun from index
const noun = this.index.getNodes().get(id)
if (!noun) {
return null
}
@ -490,7 +505,7 @@ export class BrainyData<T = any> {
return {
id,
vector: node.vector,
vector: noun.vector,
metadata
}
} catch (error) {
@ -516,7 +531,7 @@ export class BrainyData<T = any> {
}
// Remove from storage
await this.storage!.deleteNode(id)
await this.storage!.deleteNoun(id)
// Try to remove metadata (ignore errors)
try {
@ -543,11 +558,25 @@ export class BrainyData<T = any> {
try {
// Check if a vector exists
const node = this.index.getNodes().get(id)
if (!node) {
const noun = this.index.getNodes().get(id)
if (!noun) {
return false
}
// Validate noun type if metadata is for a GraphNoun
if (metadata && typeof metadata === 'object' && 'noun' in metadata) {
const nounType = (metadata as any).noun;
// Check if the noun type is valid
const isValidNounType = Object.values(NounType).includes(nounType);
if (!isValidNounType) {
console.warn(`Invalid noun type: ${nounType}. Falling back to GraphNoun.`);
// Set a default noun type
(metadata as any).noun = NounType.Concept;
}
}
// Update metadata
await this.storage!.saveMetadata(id, metadata)
@ -559,10 +588,10 @@ export class BrainyData<T = any> {
}
/**
* Add an edge between two nodes
* Add a verb between two nouns
* If metadata is provided and vector is not, the metadata will be vectorized using the embedding function
*/
public async addEdge(
public async addVerb(
sourceId: string,
targetId: string,
vector?: Vector,
@ -579,148 +608,161 @@ export class BrainyData<T = any> {
this.checkReadOnly()
try {
// Check if source and target nodes exist
const sourceNode = this.index.getNodes().get(sourceId)
const targetNode = this.index.getNodes().get(targetId)
// Check if source and target nouns exist
const sourceNoun = this.index.getNodes().get(sourceId)
const targetNoun = this.index.getNodes().get(targetId)
if (!sourceNode) {
throw new Error(`Source node with ID ${sourceId} not found`)
if (!sourceNoun) {
throw new Error(`Source noun with ID ${sourceId} not found`)
}
if (!targetNode) {
throw new Error(`Target node with ID ${targetId} not found`)
if (!targetNoun) {
throw new Error(`Target noun with ID ${targetId} not found`)
}
// Generate ID for the edge
// Generate ID for the verb
const id = uuidv4()
let edgeVector: Vector
let verbVector: Vector
// If metadata is provided and no vector is provided or forceEmbed is true, vectorize the metadata
if (options.metadata && (!vector || options.forceEmbed)) {
try {
edgeVector = await this.embeddingFunction(options.metadata)
verbVector = await this.embeddingFunction(options.metadata)
} catch (embedError) {
throw new Error(`Failed to vectorize edge metadata: ${embedError}`)
throw new Error(`Failed to vectorize verb metadata: ${embedError}`)
}
} else {
// Use a provided vector or average of source and target vectors
edgeVector =
verbVector =
vector ||
sourceNode.vector.map((val, i) => (val + targetNode.vector[i]) / 2)
sourceNoun.vector.map((val, i) => (val + targetNoun.vector[i]) / 2)
}
// Create edge
const edge: Edge = {
// Validate verb type if provided
let verbType = options.type;
if (verbType) {
// Check if the verb type is valid
const isValidVerbType = Object.values(VerbType).includes(verbType as any);
if (!isValidVerbType) {
console.warn(`Invalid verb type: ${verbType}. Using RelatedTo as default.`);
// Set a default verb type
verbType = VerbType.RelatedTo;
}
}
// Create verb
const verb: GraphVerb = {
id,
vector: edgeVector,
vector: verbVector,
connections: new Map(),
sourceId,
targetId,
type: options.type,
type: verbType,
weight: options.weight,
metadata: options.metadata
}
// Add to index
this.index.addItem({ id, vector: edgeVector })
this.index.addItem({ id, vector: verbVector })
// Get the node from the index
const indexNode = this.index.getNodes().get(id)
// Get the noun from the index
const indexNoun = this.index.getNodes().get(id)
if (!indexNode) {
if (!indexNoun) {
throw new Error(
`Failed to retrieve newly created edge node with ID ${id}`
`Failed to retrieve newly created verb noun with ID ${id}`
)
}
// Update edge connections from index
edge.connections = indexNode.connections
// Update verb connections from index
verb.connections = indexNoun.connections
// Save edge to storage
await this.storage!.saveEdge(edge)
// Save verb to storage
await this.storage!.saveVerb(verb)
return id
} catch (error) {
console.error('Failed to add edge:', error)
throw new Error(`Failed to add edge: ${error}`)
console.error('Failed to add verb:', error)
throw new Error(`Failed to add verb: ${error}`)
}
}
/**
* Get an edge by ID
* Get a verb by ID
*/
public async getEdge(id: string): Promise<Edge | null> {
public async getVerb(id: string): Promise<GraphVerb | null> {
await this.ensureInitialized()
try {
return await this.storage!.getEdge(id)
return await this.storage!.getVerb(id)
} catch (error) {
console.error(`Failed to get edge ${id}:`, error)
throw new Error(`Failed to get edge ${id}: ${error}`)
console.error(`Failed to get verb ${id}:`, error)
throw new Error(`Failed to get verb ${id}: ${error}`)
}
}
/**
* Get all edges
* Get all verbs
*/
public async getAllEdges(): Promise<Edge[]> {
public async getAllVerbs(): Promise<GraphVerb[]> {
await this.ensureInitialized()
try {
return await this.storage!.getAllEdges()
return await this.storage!.getAllVerbs()
} catch (error) {
console.error('Failed to get all edges:', error)
throw new Error(`Failed to get all edges: ${error}`)
console.error('Failed to get all verbs:', error)
throw new Error(`Failed to get all verbs: ${error}`)
}
}
/**
* Get edges by source node ID
* Get verbs by source noun ID
*/
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
try {
return await this.storage!.getEdgesBySource(sourceId)
return await this.storage!.getVerbsBySource(sourceId)
} catch (error) {
console.error(`Failed to get edges by source ${sourceId}:`, error)
throw new Error(`Failed to get edges by source ${sourceId}: ${error}`)
console.error(`Failed to get verbs by source ${sourceId}:`, error)
throw new Error(`Failed to get verbs by source ${sourceId}: ${error}`)
}
}
/**
* Get edges by target node ID
* Get verbs by target noun ID
*/
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
try {
return await this.storage!.getEdgesByTarget(targetId)
return await this.storage!.getVerbsByTarget(targetId)
} catch (error) {
console.error(`Failed to get edges by target ${targetId}:`, error)
throw new Error(`Failed to get edges by target ${targetId}: ${error}`)
console.error(`Failed to get verbs by target ${targetId}:`, error)
throw new Error(`Failed to get verbs by target ${targetId}: ${error}`)
}
}
/**
* Get edges by type
* Get verbs by type
*/
public async getEdgesByType(type: string): Promise<Edge[]> {
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
try {
return await this.storage!.getEdgesByType(type)
return await this.storage!.getVerbsByType(type)
} catch (error) {
console.error(`Failed to get edges by type ${type}:`, error)
throw new Error(`Failed to get edges by type ${type}: ${error}`)
console.error(`Failed to get verbs by type ${type}:`, error)
throw new Error(`Failed to get verbs by type ${type}: ${error}`)
}
}
/**
* Delete an edge
* Delete a verb
*/
public async deleteEdge(id: string): Promise<boolean> {
public async deleteVerb(id: string): Promise<boolean> {
await this.ensureInitialized()
// Check if database is in read-only mode
@ -734,12 +776,12 @@ export class BrainyData<T = any> {
}
// Remove from storage
await this.storage!.deleteEdge(id)
await this.storage!.deleteVerb(id)
return true
} catch (error) {
console.error(`Failed to delete edge ${id}:`, error)
throw new Error(`Failed to delete edge ${id}: ${error}`)
console.error(`Failed to delete verb ${id}:`, error)
throw new Error(`Failed to delete verb ${id}: ${error}`)
}
}

288
src/cli.ts Normal file
View file

@ -0,0 +1,288 @@
#!/usr/bin/env node
/**
* Brainy CLI
* A command-line interface for interacting with the Brainy vector database
*/
import { BrainyData, NounType, VerbType, FileSystemStorage } from './index.js';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import fs from 'fs';
import { Command } from 'commander';
// Get the directory of the current module
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// Import package.json for version info
const packageJson = {
name: '@soulcraft/brainy',
version: '0.6.0',
description: 'A vector database using HNSW indexing with Origin Private File System storage'
};
// Helper function to parse JSON safely
function parseJSON(str: string): any {
try {
return JSON.parse(str);
} catch (e) {
console.error('Error parsing JSON:', (e as Error).message);
return {};
}
}
// Helper function to resolve noun type
function resolveNounType(type: string | number | undefined): NounType {
if (!type) return NounType.Thing;
// If it's a string, try to match it to a NounType
if (typeof type === 'string') {
const nounTypeKey = Object.keys(NounType).find(
key => key.toLowerCase() === type.toLowerCase()
);
return nounTypeKey ? NounType[nounTypeKey as keyof typeof NounType] : NounType.Thing;
}
// Convert number to string type for safety
return Object.values(NounType)[type as number] || NounType.Thing;
}
// Helper function to resolve verb type
function resolveVerbType(type: string | number | undefined): VerbType {
if (!type) return VerbType.RelatedTo;
// If it's a string, try to match it to a VerbType
if (typeof type === 'string') {
const verbTypeKey = Object.keys(VerbType).find(
key => key.toLowerCase() === type.toLowerCase()
);
return verbTypeKey ? VerbType[verbTypeKey as keyof typeof VerbType] : VerbType.RelatedTo;
}
// Convert number to string type for safety
return Object.values(VerbType)[type as number] || VerbType.RelatedTo;
}
// Create a new Command instance
const program = new Command();
// Configure the program
program
.name(packageJson.name)
.description(packageJson.description)
.version(packageJson.version);
// Create data directory if it doesn't exist
const dataDir = join(__dirname, '..', 'data');
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir, { recursive: true });
}
// Create a database instance with file system storage
const createDb = () => {
return new BrainyData({
storageAdapter: new FileSystemStorage(dataDir)
});
};
// Define commands
program
.command('init')
.description('Initialize a new database')
.action(async () => {
try {
const db = createDb();
await db.init();
console.log('Database initialized successfully');
} catch (error) {
console.error('Error:', (error as Error).message);
process.exit(1);
}
});
program
.command('add')
.description('Add a new noun with the given text and optional metadata')
.argument('<text>', 'Text to add as a noun')
.argument('[metadata]', 'Optional metadata as JSON string')
.action(async (text, metadataStr) => {
try {
const db = createDb();
await db.init();
const metadata = metadataStr ? parseJSON(metadataStr) : {};
// Process metadata to handle noun type
if (metadata.noun) {
metadata.noun = resolveNounType(metadata.noun);
}
const id = await db.add(text, metadata);
console.log(`Added noun with ID: ${id}`);
} catch (error) {
console.error('Error:', (error as Error).message);
process.exit(1);
}
});
program
.command('search')
.description('Search for nouns similar to the query')
.argument('<query>', 'Search query text')
.option('-l, --limit <number>', 'Maximum number of results to return', '5')
.action(async (query, options) => {
try {
const db = createDb();
await db.init();
const limit = parseInt(options.limit, 10);
const results = await db.searchText(query, limit);
console.log(`Search results for "${query}":`);
results.forEach((result, index) => {
console.log(`${index + 1}. ID: ${result.id}`);
console.log(` Score: ${result.score.toFixed(4)}`);
console.log(` Metadata: ${JSON.stringify(result.metadata)}`);
console.log(` Vector: [${result.vector.slice(0, 3).map(v => v.toFixed(2)).join(', ')}...]`);
console.log();
});
} catch (error) {
console.error('Error:', (error as Error).message);
process.exit(1);
}
});
program
.command('get')
.description('Get a noun by ID')
.argument('<id>', 'ID of the noun to get')
.action(async (id) => {
try {
const db = createDb();
await db.init();
const noun = await db.get(id);
if (noun) {
console.log(`Noun ID: ${noun.id}`);
console.log(`Metadata: ${JSON.stringify(noun.metadata)}`);
console.log(`Vector: [${noun.vector.slice(0, 5).map(v => v.toFixed(2)).join(', ')}...]`);
} else {
console.log(`No noun found with ID: ${id}`);
}
} catch (error) {
console.error('Error:', (error as Error).message);
process.exit(1);
}
});
program
.command('delete')
.description('Delete a noun by ID')
.argument('<id>', 'ID of the noun to delete')
.action(async (id) => {
try {
const db = createDb();
await db.init();
await db.delete(id);
console.log(`Deleted noun with ID: ${id}`);
} catch (error) {
console.error('Error:', (error as Error).message);
process.exit(1);
}
});
program
.command('addVerb')
.description('Add a relationship between nouns')
.argument('<sourceId>', 'ID of the source noun')
.argument('<targetId>', 'ID of the target noun')
.argument('<verbType>', 'Type of relationship')
.argument('[metadata]', 'Optional metadata as JSON string')
.action(async (sourceId, targetId, verbTypeStr, metadataStr) => {
try {
const db = createDb();
await db.init();
const verbType = resolveVerbType(verbTypeStr);
const verbMetadata = metadataStr ? parseJSON(metadataStr) : {};
// Add verb type to metadata
verbMetadata.verb = verbType;
const verbId = await db.addVerb(sourceId, targetId, verbMetadata);
console.log(`Added verb with ID: ${verbId}`);
} catch (error) {
console.error('Error:', (error as Error).message);
process.exit(1);
}
});
program
.command('getVerbs')
.description('Get all relationships for a noun')
.argument('<id>', 'ID of the noun to get relationships for')
.action(async (id) => {
try {
const db = createDb();
await db.init();
const verbs = await db.getVerbsBySource(id);
console.log(`Relationships for noun ${id}:`);
if (verbs.length === 0) {
console.log('No relationships found');
} else {
verbs.forEach((verb, index) => {
console.log(`${index + 1}. ID: ${verb.id}`);
console.log(` Type: ${Object.keys(VerbType).find(key => VerbType[key as keyof typeof VerbType] === verb.metadata.verb) || verb.metadata.verb}`);
console.log(` Target: ${verb.targetId}`);
console.log(` Metadata: ${JSON.stringify(verb.metadata)}`);
console.log();
});
}
} catch (error) {
console.error('Error:', (error as Error).message);
process.exit(1);
}
});
program
.command('status')
.description('Show database status')
.action(async () => {
try {
const db = createDb();
await db.init();
const status = await db.status();
console.log('Database Status:');
console.log(`Storage type: ${status.type}`);
console.log(`Storage used: ${status.used} bytes`);
console.log(`Storage quota: ${status.quota !== null ? `${status.quota} bytes` : 'unlimited'}`);
// Display additional details if available
if (status.details) {
console.log('Additional details:');
Object.entries(status.details).forEach(([key, value]) => {
console.log(` ${key}: ${JSON.stringify(value)}`);
});
}
} catch (error) {
console.error('Error:', (error as Error).message);
process.exit(1);
}
});
// Add examples to help text
program.addHelpText('after', `
Examples:
$ brainy init
$ brainy add "Cats are independent pets" '{"noun":"Thing","category":"animal"}'
$ brainy search "feline pets" --limit 5
$ brainy addVerb id1 id2 RelatedTo '{"description":"Both are pets"}'
`);
// Parse command line arguments
program.parse();

48
src/coreTypes.ts
View file

@ -57,31 +57,31 @@ export interface EmbeddingModel {
}
/**
* HNSW graph node
* HNSW graph noun
*/
export interface HNSWNode {
export interface HNSWNoun {
id: string;
vector: Vector;
connections: Map<number, Set<string>>; // level -> set of connected node ids
connections: Map<number, Set<string>>; // level -> set of connected noun ids
}
/**
* Edge representing a relationship between nodes
* Extends HNSWNode to allow edges to be first-class entities in the data model
* Verb representing a relationship between nouns
* Extends HNSWNoun to allow verbs to be first-class entities in the data model
*/
export interface Edge extends HNSWNode {
sourceId: string; // ID of the source node
targetId: string; // ID of the target node
export interface GraphVerb extends HNSWNoun {
sourceId: string; // ID of the source noun
targetId: string; // ID of the target noun
type?: string; // Optional type of the relationship
weight?: number; // Optional weight of the relationship
metadata?: any; // Optional metadata for the edge
metadata?: any; // Optional metadata for the verb
}
/**
* HNSW index configuration
*/
export interface HNSWConfig {
M: number; // Maximum number of connections per node
M: number; // Maximum number of connections per noun
efConstruction: number; // Size of the dynamic candidate list during construction
efSearch: number; // Size of the dynamic candidate list during search
ml: number; // Maximum level
@ -93,34 +93,34 @@ export interface HNSWConfig {
export interface StorageAdapter {
init(): Promise<void>;
saveNode(node: HNSWNode): Promise<void>;
saveNoun(noun: HNSWNoun): Promise<void>;
getNode(id: string): Promise<HNSWNode | null>;
getNoun(id: string): Promise<HNSWNoun | null>;
getAllNodes(): Promise<HNSWNode[]>;
getAllNouns(): Promise<HNSWNoun[]>;
/**
* Get nodes by noun type
* Get nouns by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
* @returns Promise that resolves to an array of nouns of the specified noun type
*/
getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
getNounsByNounType(nounType: string): Promise<HNSWNoun[]>;
deleteNode(id: string): Promise<void>;
deleteNoun(id: string): Promise<void>;
saveEdge(edge: Edge): Promise<void>;
saveVerb(verb: GraphVerb): Promise<void>;
getEdge(id: string): Promise<Edge | null>;
getVerb(id: string): Promise<GraphVerb | null>;
getAllEdges(): Promise<Edge[]>;
getAllVerbs(): Promise<GraphVerb[]>;
getEdgesBySource(sourceId: string): Promise<Edge[]>;
getVerbsBySource(sourceId: string): Promise<GraphVerb[]>;
getEdgesByTarget(targetId: string): Promise<Edge[]>;
getVerbsByTarget(targetId: string): Promise<GraphVerb[]>;
getEdgesByType(type: string): Promise<Edge[]>;
getVerbsByType(type: string): Promise<GraphVerb[]>;
deleteEdge(id: string): Promise<void>;
deleteVerb(id: string): Promise<void>;
saveMetadata(id: string, metadata: any): Promise<void>;

184
src/hnsw/hnswIndex.ts
View file

@ -3,19 +3,19 @@
* Based on the paper: "Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs"
*/
import { DistanceFunction, HNSWConfig, HNSWNode, Vector, VectorDocument } from '../coreTypes.js'
import { DistanceFunction, HNSWConfig, HNSWNoun, Vector, VectorDocument } from '../coreTypes.js'
import { euclideanDistance } from '../utils/index.js'
// Default HNSW parameters
const DEFAULT_CONFIG: HNSWConfig = {
M: 16, // Max number of connections per node
M: 16, // Max number of connections per noun
efConstruction: 200, // Size of a dynamic candidate list during construction
efSearch: 50, // Size of a dynamic candidate list during search
ml: 16 // Max level
}
export class HNSWIndex {
private nodes: Map<string, HNSWNode> = new Map()
private nouns: Map<string, HNSWNoun> = new Map()
private entryPointId: string | null = null
private maxLevel = 0
private config: HNSWConfig
@ -55,55 +55,55 @@ export class HNSWIndex {
)
}
// Generate random level for this node
const nodeLevel = this.getRandomLevel()
// Generate random level for this noun
const nounLevel = this.getRandomLevel()
// Create new node
const node: HNSWNode = {
// Create new noun
const noun: HNSWNoun = {
id,
vector,
connections: new Map()
}
// Initialize empty connection sets for each level
for (let level = 0; level <= nodeLevel; level++) {
node.connections.set(level, new Set<string>())
for (let level = 0; level <= nounLevel; level++) {
noun.connections.set(level, new Set<string>())
}
// If this is the first node, make it the entry point
if (this.nodes.size === 0) {
// If this is the first noun, make it the entry point
if (this.nouns.size === 0) {
this.entryPointId = id
this.maxLevel = nodeLevel
this.nodes.set(id, node)
this.maxLevel = nounLevel
this.nouns.set(id, noun)
return id
}
// Find entry point
if (!this.entryPointId) {
console.error('Entry point ID is null')
// If there's no entry point, this is the first node, so we should have returned earlier
// If there's no entry point, this is the first noun, so we should have returned earlier
// This is a safety check
this.entryPointId = id
this.maxLevel = nodeLevel
this.nodes.set(id, node)
this.maxLevel = nounLevel
this.nouns.set(id, noun)
return id
}
const entryPoint = this.nodes.get(this.entryPointId)
const entryPoint = this.nouns.get(this.entryPointId)
if (!entryPoint) {
console.error(`Entry point with ID ${this.entryPointId} not found`)
// If the entry point doesn't exist, treat this as the first node
// If the entry point doesn't exist, treat this as the first noun
this.entryPointId = id
this.maxLevel = nodeLevel
this.nodes.set(id, node)
this.maxLevel = nounLevel
this.nouns.set(id, noun)
return id
}
let currObj = entryPoint
let currDist = this.distanceFunction(vector, entryPoint.vector)
// Traverse the graph from top to bottom to find the closest node
for (let level = this.maxLevel; level > nodeLevel; level--) {
// Traverse the graph from top to bottom to find the closest noun
for (let level = this.maxLevel; level > nounLevel; level--) {
let changed = true
while (changed) {
changed = false
@ -112,7 +112,7 @@ export class HNSWIndex {
const connections = currObj.connections.get(level) || new Set<string>()
for (const neighborId of connections) {
const neighbor = this.nodes.get(neighborId)
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in addItem traversal`)
continue
@ -128,10 +128,10 @@ export class HNSWIndex {
}
}
// For each level from nodeLevel down to 0
for (let level = Math.min(nodeLevel, this.maxLevel); level >= 0; level--) {
// For each level from nounLevel down to 0
for (let level = Math.min(nounLevel, this.maxLevel); level >= 0; level--) {
// Find ef nearest elements using greedy search
const nearestNodes = this.searchLayer(
const nearestNouns = this.searchLayer(
vector,
currObj,
this.config.efConstruction,
@ -141,19 +141,19 @@ export class HNSWIndex {
// Select M nearest neighbors
const neighbors = this.selectNeighbors(
vector,
nearestNodes,
nearestNouns,
this.config.M
)
// Add bidirectional connections
for (const [neighborId, _] of neighbors) {
const neighbor = this.nodes.get(neighborId)
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found`)
continue
}
node.connections.get(level)!.add(neighborId)
noun.connections.get(level)!.add(neighborId)
// Add reverse connection
if (!neighbor.connections.has(level)) {
@ -168,29 +168,29 @@ export class HNSWIndex {
}
// Update entry point for the next level
if (nearestNodes.size > 0) {
const [nearestId, nearestDist] = [...nearestNodes][0]
if (nearestNouns.size > 0) {
const [nearestId, nearestDist] = [...nearestNouns][0]
if (nearestDist < currDist) {
currDist = nearestDist
const nearestNode = this.nodes.get(nearestId)
if (!nearestNode) {
console.error(`Nearest node with ID ${nearestId} not found in addItem`)
const nearestNoun = this.nouns.get(nearestId)
if (!nearestNoun) {
console.error(`Nearest noun with ID ${nearestId} not found in addItem`)
// Keep the current object as is
} else {
currObj = nearestNode
currObj = nearestNoun
}
}
}
}
// Update max level and entry point if needed
if (nodeLevel > this.maxLevel) {
this.maxLevel = nodeLevel
if (nounLevel > this.maxLevel) {
this.maxLevel = nounLevel
this.entryPointId = id
}
// Add node to the index
this.nodes.set(id, node)
// Add noun to the index
this.nouns.set(id, noun)
return id
}
@ -198,7 +198,7 @@ export class HNSWIndex {
* Search for nearest neighbors
*/
public search(queryVector: Vector, k: number = 10): Array<[string, number]> {
if (this.nodes.size === 0) {
if (this.nouns.size === 0) {
return []
}
@ -219,7 +219,7 @@ export class HNSWIndex {
return []
}
const entryPoint = this.nodes.get(this.entryPointId)
const entryPoint = this.nouns.get(this.entryPointId)
if (!entryPoint) {
console.error(`Entry point with ID ${this.entryPointId} not found`)
return []
@ -228,7 +228,7 @@ export class HNSWIndex {
let currObj = entryPoint
let currDist = this.distanceFunction(queryVector, currObj.vector)
// Traverse the graph from top to bottom to find the closest node
// Traverse the graph from top to bottom to find the closest noun
for (let level = this.maxLevel; level > 0; level--) {
let changed = true
while (changed) {
@ -238,7 +238,7 @@ export class HNSWIndex {
const connections = currObj.connections.get(level) || new Set<string>()
for (const neighborId of connections) {
const neighbor = this.nodes.get(neighborId)
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in search`)
continue
@ -258,7 +258,7 @@ export class HNSWIndex {
}
// Search at level 0 with ef = k
const nearestNodes = this.searchLayer(
const nearestNouns = this.searchLayer(
queryVector,
currObj,
Math.max(this.config.efSearch, k),
@ -266,23 +266,23 @@ export class HNSWIndex {
)
// Convert to array and sort by distance
return [...nearestNodes].slice(0, k)
return [...nearestNouns].slice(0, k)
}
/**
* Remove an item from the index
*/
public removeItem(id: string): boolean {
if (!this.nodes.has(id)) {
if (!this.nouns.has(id)) {
return false
}
const node = this.nodes.get(id)!
const noun = this.nouns.get(id)!
// Remove connections to this node from all neighbors
for (const [level, connections] of node.connections.entries()) {
// Remove connections to this noun from all neighbors
for (const [level, connections] of noun.connections.entries()) {
for (const neighborId of connections) {
const neighbor = this.nodes.get(neighborId)
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in removeItem`)
continue
@ -290,46 +290,46 @@ export class HNSWIndex {
if (neighbor.connections.has(level)) {
neighbor.connections.get(level)!.delete(id)
// Prune connections after removing this node to ensure consistency
// Prune connections after removing this noun to ensure consistency
this.pruneConnections(neighbor, level)
}
}
}
// Also check all other nodes for references to this node and remove them
for (const [nodeId, otherNode] of this.nodes.entries()) {
if (nodeId === id) continue // Skip the node being removed
// Also check all other nouns for references to this noun and remove them
for (const [nounId, otherNoun] of this.nouns.entries()) {
if (nounId === id) continue // Skip the noun being removed
for (const [level, connections] of otherNode.connections.entries()) {
for (const [level, connections] of otherNoun.connections.entries()) {
if (connections.has(id)) {
connections.delete(id)
// Prune connections after removing this reference
this.pruneConnections(otherNode, level)
this.pruneConnections(otherNoun, level)
}
}
}
// Remove the node
this.nodes.delete(id)
// Remove the noun
this.nouns.delete(id)
// If we removed the entry point, find a new one
if (this.entryPointId === id) {
if (this.nodes.size === 0) {
if (this.nouns.size === 0) {
this.entryPointId = null
this.maxLevel = 0
} else {
// Find the node with the highest level
// Find the noun with the highest level
let maxLevel = 0
let newEntryPointId = null
for (const [nodeId, node] of this.nodes.entries()) {
if (node.connections.size === 0) continue // Skip nodes with no connections
for (const [nounId, noun] of this.nouns.entries()) {
if (noun.connections.size === 0) continue // Skip nouns with no connections
const nodeLevel = Math.max(...node.connections.keys())
if (nodeLevel >= maxLevel) {
maxLevel = nodeLevel
newEntryPointId = nodeId
const nounLevel = Math.max(...noun.connections.keys())
if (nounLevel >= maxLevel) {
maxLevel = nounLevel
newEntryPointId = nounId
}
}
@ -342,17 +342,25 @@ export class HNSWIndex {
}
/**
* Get all nodes in the index
* Get all nouns in the index
*/
public getNodes(): Map<string, HNSWNode> {
return new Map(this.nodes)
public getNouns(): Map<string, HNSWNoun> {
return new Map(this.nouns)
}
/**
* Get all nodes in the index (alias for getNouns for backward compatibility)
* @deprecated Use getNouns() instead
*/
public getNodes(): Map<string, HNSWNoun> {
return this.getNouns()
}
/**
* Clear the index
*/
public clear(): void {
this.nodes.clear()
this.nouns.clear()
this.entryPointId = null
this.maxLevel = 0
}
@ -361,7 +369,7 @@ export class HNSWIndex {
* Get the size of the index
*/
public size(): number {
return this.nodes.size
return this.nouns.size
}
/**
@ -373,15 +381,15 @@ export class HNSWIndex {
/**
* Search within a specific layer
* Returns a map of node IDs to distances, sorted by distance
* Returns a map of noun IDs to distances, sorted by distance
*/
private searchLayer(
queryVector: Vector,
entryPoint: HNSWNode,
entryPoint: HNSWNoun,
ef: number,
level: number
): Map<string, number> {
// Set of visited nodes
// Set of visited nouns
const visited = new Set<string>([entryPoint.id])
// Priority queue of candidates (closest first)
@ -411,18 +419,18 @@ export class HNSWIndex {
}
// Explore neighbors of the closest candidate
const node = this.nodes.get(closestId)
if (!node) {
console.error(`Node with ID ${closestId} not found in searchLayer`)
const noun = this.nouns.get(closestId)
if (!noun) {
console.error(`Noun with ID ${closestId} not found in searchLayer`)
continue
}
const connections = node.connections.get(level) || new Set<string>()
const connections = noun.connections.get(level) || new Set<string>()
for (const neighborId of connections) {
if (!visited.has(neighborId)) {
visited.add(neighborId)
const neighbor = this.nodes.get(neighborId)
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in searchLayer`)
continue
@ -478,10 +486,10 @@ export class HNSWIndex {
}
/**
* Ensure a node doesn't have too many connections at a given level
* Ensure a noun doesn't have too many connections at a given level
*/
private pruneConnections(node: HNSWNode, level: number): void {
const connections = node.connections.get(level)!
private pruneConnections(noun: HNSWNoun, level: number): void {
const connections = noun.connections.get(level)!
if (connections.size <= this.config.M) {
return
}
@ -491,7 +499,7 @@ export class HNSWIndex {
const validNeighborIds = new Set<string>()
for (const neighborId of connections) {
const neighbor = this.nodes.get(neighborId)
const neighbor = this.nouns.get(neighborId)
if (!neighbor) {
console.error(`Neighbor with ID ${neighborId} not found in pruneConnections`)
continue
@ -500,7 +508,7 @@ export class HNSWIndex {
// Only add valid neighbors to the distances map
distances.set(
neighborId,
this.distanceFunction(node.vector, neighbor.vector)
this.distanceFunction(noun.vector, neighbor.vector)
)
validNeighborIds.add(neighborId)
}
@ -508,23 +516,23 @@ export class HNSWIndex {
// Only proceed if we have valid neighbors
if (distances.size === 0) {
// If no valid neighbors, clear connections at this level
node.connections.set(level, new Set())
noun.connections.set(level, new Set())
return
}
// Select M closest neighbors from valid ones
const selectedNeighbors = this.selectNeighbors(
node.vector,
noun.vector,
distances,
this.config.M
)
// Update connections with only valid neighbors
node.connections.set(level, new Set(selectedNeighbors.keys()))
noun.connections.set(level, new Set(selectedNeighbors.keys()))
}
/**
* Generate a random level for a new node
* Generate a random level for a new noun
* Uses the same distribution as in the original HNSW paper
*/
private getRandomLevel(): number {

10
src/index.ts
View file

@ -148,8 +148,8 @@ import type {
DistanceFunction,
EmbeddingFunction,
EmbeddingModel,
HNSWNode,
Edge,
HNSWNoun,
GraphVerb,
HNSWConfig,
StorageAdapter
} from './coreTypes.js'
@ -161,8 +161,8 @@ export type {
DistanceFunction,
EmbeddingFunction,
EmbeddingModel,
HNSWNode,
Edge,
HNSWNoun,
GraphVerb,
HNSWConfig,
StorageAdapter
}
@ -213,7 +213,6 @@ export type {
// Export graph types
import type {
GraphNoun,
GraphVerb,
EmbeddedGraphVerb,
Person,
Place,
@ -226,7 +225,6 @@ import { NounType, VerbType } from './types/graphTypes.js'
export type {
GraphNoun,
GraphVerb,
EmbeddedGraphVerb,
Person,
Place,

68
src/storage/fileSystemStorage.ts
View file

@ -1,4 +1,4 @@
import { Edge, HNSWNode, StorageAdapter } from '../coreTypes.js'
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
// We'll dynamically import Node.js built-in modules
let fs: any
@ -114,35 +114,35 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Save a node to storage
*/
public async saveNode(node: HNSWNode): Promise<void> {
public async saveNoun(noun: HNSWNoun): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableNode = {
...node,
connections: this.mapToObject(node.connections, (set) => Array.from(set))
...noun,
connections: this.mapToObject(noun.connections, (set) => Array.from(set as Set<string>))
}
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(node.id)
const nodeDir = await this.getNodeDirectory(noun.id)
const filePath = path.join(nodeDir, `${node.id}.json`)
const filePath = path.join(nodeDir, `${noun.id}.json`)
await fs.promises.writeFile(
filePath,
JSON.stringify(serializableNode, null, 2),
'utf8'
)
} catch (error) {
console.error(`Failed to save node ${node.id}:`, error)
throw new Error(`Failed to save node ${node.id}: ${error}`)
console.error(`Failed to save node ${noun.id}:`, error)
throw new Error(`Failed to save node ${noun.id}: ${error}`)
}
}
/**
* Get a node from storage
*/
public async getNode(id: string): Promise<HNSWNode | null> {
public async getNoun(id: string): Promise<HNSWNoun | null> {
await this.ensureInitialized()
try {
@ -243,7 +243,7 @@ export class FileSystemStorage implements StorageAdapter {
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
await this.ensureInitialized()
try {
@ -272,7 +272,7 @@ export class FileSystemStorage implements StorageAdapter {
dir = this.defaultDir
}
const nodes: HNSWNode[] = []
const nodes: HNSWNoun[] = []
try {
const files = await fs.promises.readdir(dir)
@ -284,7 +284,7 @@ export class FileSystemStorage implements StorageAdapter {
})
const dirNodes = await Promise.all(nodePromises)
nodes.push(...dirNodes.filter((node): node is HNSWNode => node !== null))
nodes.push(...dirNodes.filter((node): node is HNSWNoun => node !== null))
} catch (dirError) {
// If directory doesn't exist or can't be read, log a warning
console.warn(`Could not read directory for noun type ${nounType}:`, dirError)
@ -300,7 +300,7 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Get all nodes from storage
*/
public async getAllNodes(): Promise<HNSWNode[]> {
public async getAllNouns(): Promise<HNSWNoun[]> {
await this.ensureInitialized()
try {
@ -316,11 +316,11 @@ export class FileSystemStorage implements StorageAdapter {
]
// Run searches in parallel for all noun types
const nodePromises = nounTypes.map(nounType => this.getNodesByNounType(nounType))
const nodePromises = nounTypes.map(nounType => this.getNounsByNounType(nounType))
const nodeArrays = await Promise.all(nodePromises)
// Combine all results
const allNodes: HNSWNode[] = []
const allNodes: HNSWNoun[] = []
for (const nodes of nodeArrays) {
allNodes.push(...nodes)
}
@ -335,7 +335,7 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Read a node from a file
*/
private async readNodeFromFile(filePath: string): Promise<HNSWNode | null> {
private async readNodeFromFile(filePath: string): Promise<HNSWNoun | null> {
try {
const data = await fs.promises.readFile(filePath, 'utf8')
const parsedNode = JSON.parse(data)
@ -360,7 +360,7 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Delete a node from storage
*/
public async deleteNode(id: string): Promise<void> {
public async deleteNoun(id: string): Promise<void> {
await this.ensureInitialized()
try {
@ -420,32 +420,32 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Save an edge to storage
*/
public async saveEdge(edge: Edge): Promise<void> {
public async saveVerb(verb: GraphVerb): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableEdge = {
...edge,
connections: this.mapToObject(edge.connections, (set) => Array.from(set))
...verb,
connections: this.mapToObject(verb.connections, (set) => Array.from(set as Set<string>))
}
const filePath = path.join(this.edgesDir, `${edge.id}.json`)
const filePath = path.join(this.edgesDir, `${verb.id}.json`)
await fs.promises.writeFile(
filePath,
JSON.stringify(serializableEdge, null, 2),
'utf8'
)
} catch (error) {
console.error(`Failed to save edge ${edge.id}:`, error)
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
console.error(`Failed to save edge ${verb.id}:`, error)
throw new Error(`Failed to save edge ${verb.id}: ${error}`)
}
}
/**
* Get an edge from storage
*/
public async getEdge(id: string): Promise<Edge | null> {
public async getVerb(id: string): Promise<GraphVerb | null> {
await this.ensureInitialized()
try {
@ -486,7 +486,7 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Get all edges from storage
*/
public async getAllEdges(): Promise<Edge[]> {
public async getAllVerbs(): Promise<GraphVerb[]> {
await this.ensureInitialized()
try {
@ -495,11 +495,11 @@ export class FileSystemStorage implements StorageAdapter {
.filter((file: string) => file.endsWith('.json'))
.map((file: string) => {
const id = path.basename(file, '.json')
return this.getEdge(id)
return this.getVerb(id)
})
const edges = await Promise.all(edgePromises)
return edges.filter((edge): edge is Edge => edge !== null)
return edges.filter((edge): edge is GraphVerb => edge !== null)
} catch (error) {
console.error('Failed to get all edges:', error)
throw new Error(`Failed to get all edges: ${error}`)
@ -509,11 +509,11 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Get edges by source node ID
*/
public async getEdgesBySource(sourceId: string): Promise<Edge[]> {
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
const allEdges = await this.getAllVerbs()
return allEdges.filter(edge => edge.sourceId === sourceId)
} catch (error) {
console.error(`Failed to get edges by source ${sourceId}:`, error)
@ -524,11 +524,11 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Get edges by target node ID
*/
public async getEdgesByTarget(targetId: string): Promise<Edge[]> {
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
const allEdges = await this.getAllVerbs()
return allEdges.filter(edge => edge.targetId === targetId)
} catch (error) {
console.error(`Failed to get edges by target ${targetId}:`, error)
@ -539,11 +539,11 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Get edges by type
*/
public async getEdgesByType(type: string): Promise<Edge[]> {
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
await this.ensureInitialized()
try {
const allEdges = await this.getAllEdges()
const allEdges = await this.getAllVerbs()
return allEdges.filter(edge => edge.type === type)
} catch (error) {
console.error(`Failed to get edges by type ${type}:`, error)
@ -554,7 +554,7 @@ export class FileSystemStorage implements StorageAdapter {
/**
* Delete an edge from storage
*/
public async deleteEdge(id: string): Promise<void> {
public async deleteVerb(id: string): Promise<void> {
await this.ensureInitialized()
try {

220
src/storage/opfsStorage.ts
View file

@ -3,7 +3,11 @@
* Provides persistent storage for the vector database using the Origin Private File System API
*/
import { Edge, HNSWNode, StorageAdapter } from '../coreTypes.js'
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
// Type aliases for compatibility
type HNSWNode = HNSWNoun;
type Edge = GraphVerb;
// Directory and file names
const ROOT_DIR = 'opfs-vector-db'
@ -165,42 +169,42 @@ export class OPFSStorage implements StorageAdapter {
}
/**
* Save a node to storage
* Save a noun to storage
*/
public async saveNode(node: HNSWNode): Promise<void> {
public async saveNoun(noun: HNSWNoun): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableNode = {
...node,
connections: this.mapToObject(node.connections, (set) =>
Array.from(set)
const serializableNoun = {
...noun,
connections: this.mapToObject(noun.connections, (set) =>
Array.from(set as Set<string>)
)
}
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(node.id)
// Get the appropriate directory based on the noun's metadata
const nounDir = await this.getNodeDirectory(noun.id)
// Create or get the file for this node
const fileHandle = await nodeDir.getFileHandle(node.id, {
// Create or get the file for this noun
const fileHandle = await nounDir.getFileHandle(noun.id, {
create: true
})
// Write the node data to the file
// Write the noun data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableNode))
await writable.write(JSON.stringify(serializableNoun))
await writable.close()
} catch (error) {
console.error(`Failed to save node ${node.id}:`, error)
throw new Error(`Failed to save node ${node.id}: ${error}`)
console.error(`Failed to save noun ${noun.id}:`, error)
throw new Error(`Failed to save noun ${noun.id}: ${error}`)
}
}
/**
* Get a node from storage
* Get a noun from storage
*/
public async getNode(id: string): Promise<HNSWNode | null> {
public async getNoun(id: string): Promise<HNSWNoun | null> {
await this.ensureInitialized()
try {
@ -301,15 +305,15 @@ export class OPFSStorage implements StorageAdapter {
}
/**
* Get nodes by noun type
* Get nouns by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
* @returns Promise that resolves to an array of nouns of the specified noun type
*/
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
await this.ensureInitialized()
try {
const nodes: HNSWNode[] = []
const nouns: HNSWNoun[] = []
// Determine the directory based on the noun type
let dir: FileSystemDirectoryHandle
@ -358,7 +362,7 @@ export class OPFSStorage implements StorageAdapter {
connections.set(Number(level), new Set(nodeIds as string[]))
}
nodes.push({
nouns.push({
id: data.id,
vector: data.vector,
connections
@ -372,17 +376,17 @@ export class OPFSStorage implements StorageAdapter {
console.warn(`Failed to read directory for noun type ${nounType}:`, dirError)
}
return nodes
return nouns
} catch (error) {
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
console.error(`Failed to get nouns for noun type ${nounType}:`, error)
throw new Error(`Failed to get nouns for noun type ${nounType}: ${error}`)
}
}
/**
* Get all nodes from storage
* Get all nouns from storage
*/
public async getAllNodes(): Promise<HNSWNode[]> {
public async getAllNouns(): Promise<HNSWNoun[]> {
await this.ensureInitialized()
try {
@ -398,42 +402,42 @@ export class OPFSStorage implements StorageAdapter {
]
// Run searches in parallel for all noun types
const nodePromises = nounTypes.map(nounType => this.getNodesByNounType(nounType))
const nodeArrays = await Promise.all(nodePromises)
const nounPromises = nounTypes.map(nounType => this.getNounsByNounType(nounType))
const nounArrays = await Promise.all(nounPromises)
// Combine all results
const allNodes: HNSWNode[] = []
for (const nodes of nodeArrays) {
allNodes.push(...nodes)
const allNouns: HNSWNoun[] = []
for (const nouns of nounArrays) {
allNouns.push(...nouns)
}
return allNodes
return allNouns
} catch (error) {
console.error('Failed to get all nodes:', error)
throw new Error(`Failed to get all nodes: ${error}`)
console.error('Failed to get all nouns:', error)
throw new Error(`Failed to get all nouns: ${error}`)
}
}
/**
* Delete a node from storage
* Delete a noun from storage
*/
public async deleteNode(id: string): Promise<void> {
public async deleteNoun(id: string): Promise<void> {
await this.ensureInitialized()
try {
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(id)
// Get the appropriate directory based on the noun's metadata
const nounDir = await this.getNodeDirectory(id)
try {
// Try to delete the node from the appropriate directory
await nodeDir.removeEntry(id)
return // Node deleted successfully
// Try to delete the noun from the appropriate directory
await nounDir.removeEntry(id)
return // Noun deleted successfully
} catch (dirError) {
// If the file doesn't exist in the expected directory, try the default directory
if (nodeDir !== this.defaultDir) {
if (nounDir !== this.defaultDir) {
try {
await this.defaultDir!.removeEntry(id)
return // Node deleted successfully
return // Noun deleted successfully
} catch (defaultDirError) {
// If not found in default directory either, try all noun type directories
const directories = [
@ -446,7 +450,7 @@ export class OPFSStorage implements StorageAdapter {
]
for (const dir of directories) {
if (dir === nodeDir) continue // Skip the already checked directory
if (dir === nounDir) continue // Skip the already checked directory
try {
await dir.removeEntry(id)
@ -464,52 +468,52 @@ export class OPFSStorage implements StorageAdapter {
return
}
} catch (error) {
console.error(`Failed to delete node ${id}:`, error)
throw new Error(`Failed to delete node ${id}: ${error}`)
console.error(`Failed to delete noun ${id}:`, error)
throw new Error(`Failed to delete noun ${id}: ${error}`)
}
}
/**
* Save an edge to storage
* Save a verb to storage
*/
public async saveEdge(edge: Edge): Promise<void> {
public async saveVerb(verb: GraphVerb): Promise<void> {
await this.ensureInitialized()
try {
// Convert connections Map to a serializable format
const serializableEdge = {
...edge,
connections: this.mapToObject(edge.connections, (set) =>
Array.from(set)
const serializableVerb = {
...verb,
connections: this.mapToObject(verb.connections, (set) =>
Array.from(set as Set<string>)
)
}
// Create or get the file for this edge
const fileHandle = await this.edgesDir!.getFileHandle(edge.id, {
// Create or get the file for this verb
const fileHandle = await this.edgesDir!.getFileHandle(verb.id, {
create: true
})
// Write the edge data to the file
// Write the verb data to the file
const writable = await fileHandle.createWritable()
await writable.write(JSON.stringify(serializableEdge))
await writable.write(JSON.stringify(serializableVerb))
await writable.close()
} catch (error) {
console.error(`Failed to save edge ${edge.id}:`, error)
throw new Error(`Failed to save edge ${edge.id}: ${error}`)
console.error(`Failed to save verb ${verb.id}:`, error)
throw new Error(`Failed to save verb ${verb.id}: ${error}`)
}
}
/**
* Get an edge from storage
* Get a verb from storage
*/
public async getEdge(id: string): Promise<Edge | null> {
public async getVerb(id: string): Promise<GraphVerb | null> {
await this.ensureInitialized()
try {
// Get the file handle for this edge
// Get the file handle for this verb
const fileHandle = await this.edgesDir!.getFileHandle(id)
// Read the edge data from the file
// Read the verb data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
@ -536,8 +540,8 @@ export class OPFSStorage implements StorageAdapter {
return null
}
console.error(`Failed to get edge ${id}:`, error)
throw new Error(`Failed to get edge ${id}: ${error}`)
console.error(`Failed to get verb ${id}:`, error)
throw new Error(`Failed to get verb ${id}: ${error}`)
}
}
@ -556,7 +560,7 @@ export class OPFSStorage implements StorageAdapter {
// Iterate through all keys and get the corresponding edges
for await (const name of keys) {
const edge = await this.getEdge(name)
const edge = await this.getVerb(name)
if (edge) {
edges.push(edge)
}
@ -569,6 +573,13 @@ export class OPFSStorage implements StorageAdapter {
}
}
/**
* Get all verbs from storage (alias for getAllEdges)
*/
public async getAllVerbs(): Promise<GraphVerb[]> {
return this.getAllEdges();
}
/**
* Delete an edge from storage
*/
@ -586,6 +597,13 @@ export class OPFSStorage implements StorageAdapter {
}
}
/**
* Delete a verb from storage (alias for deleteEdge)
*/
public async deleteVerb(id: string): Promise<void> {
return this.deleteEdge(id);
}
/**
* Get edges by source node ID
*/
@ -601,6 +619,13 @@ export class OPFSStorage implements StorageAdapter {
}
}
/**
* Get verbs by source node ID (alias for getEdgesBySource)
*/
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
return this.getEdgesBySource(sourceId);
}
/**
* Get edges by target node ID
*/
@ -616,6 +641,13 @@ export class OPFSStorage implements StorageAdapter {
}
}
/**
* Get verbs by target node ID (alias for getEdgesByTarget)
*/
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
return this.getEdgesByTarget(targetId);
}
/**
* Get edges by type
*/
@ -631,6 +663,13 @@ export class OPFSStorage implements StorageAdapter {
}
}
/**
* Get verbs by type (alias for getEdgesByType)
*/
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
return this.getEdgesByType(type);
}
/**
* Save metadata for a node
*/
@ -949,6 +988,55 @@ export class MemoryStorage implements StorageAdapter {
this.nodes.set(DEFAULT_DIR, new Map())
}
// Alias methods to match StorageAdapter interface
public async saveNoun(noun: HNSWNoun): Promise<void> {
return this.saveNode(noun);
}
public async getNoun(id: string): Promise<HNSWNoun | null> {
return this.getNode(id);
}
public async getAllNouns(): Promise<HNSWNoun[]> {
return this.getAllNodes();
}
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
return this.getNodesByNounType(nounType);
}
public async deleteNoun(id: string): Promise<void> {
return this.deleteNode(id);
}
public async saveVerb(verb: GraphVerb): Promise<void> {
return this.saveEdge(verb);
}
public async getVerb(id: string): Promise<GraphVerb | null> {
return this.getEdge(id);
}
public async getAllVerbs(): Promise<GraphVerb[]> {
return this.getAllEdges();
}
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
return this.getEdgesBySource(sourceId);
}
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
return this.getEdgesByTarget(targetId);
}
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
return this.getEdgesByType(type);
}
public async deleteVerb(id: string): Promise<void> {
return this.deleteEdge(id);
}
public async init(): Promise<void> {
// Nothing to initialize for in-memory storage
}

59
src/storage/s3CompatibleStorage.ts
View file

@ -1,4 +1,8 @@
import { Edge, HNSWNode, StorageAdapter } from '../coreTypes.js'
import { GraphVerb, HNSWNoun, StorageAdapter } from '../coreTypes.js'
// Type aliases for compatibility
type HNSWNode = HNSWNoun;
type Edge = GraphVerb;
// Constants for S3 bucket prefixes
const NODES_PREFIX = 'nodes/'
@ -59,6 +63,55 @@ export class S3CompatibleStorage implements StorageAdapter {
private s3Client: any // Will be initialized in init()
private isInitialized = false
// Alias methods to match StorageAdapter interface
public async saveNoun(noun: HNSWNoun): Promise<void> {
return this.saveNode(noun);
}
public async getNoun(id: string): Promise<HNSWNoun | null> {
return this.getNode(id);
}
public async getAllNouns(): Promise<HNSWNoun[]> {
return this.getAllNodes();
}
public async getNounsByNounType(nounType: string): Promise<HNSWNoun[]> {
return this.getNodesByNounType(nounType);
}
public async deleteNoun(id: string): Promise<void> {
return this.deleteNode(id);
}
public async saveVerb(verb: GraphVerb): Promise<void> {
return this.saveEdge(verb);
}
public async getVerb(id: string): Promise<GraphVerb | null> {
return this.getEdge(id);
}
public async getAllVerbs(): Promise<GraphVerb[]> {
return this.getAllEdges();
}
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
return this.getEdgesBySource(sourceId);
}
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
return this.getEdgesByTarget(targetId);
}
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
return this.getEdgesByType(type);
}
public async deleteVerb(id: string): Promise<void> {
return this.deleteEdge(id);
}
constructor(options: {
bucketName: string
accessKeyId: string
@ -161,7 +214,7 @@ export class S3CompatibleStorage implements StorageAdapter {
// Convert connections Map to a serializable format
const serializableNode = {
...node,
connections: this.mapToObject(node.connections, (set) => Array.from(set))
connections: this.mapToObject(node.connections, (set) => Array.from(set as Set<string>))
}
// Get the appropriate prefix based on the node's metadata
@ -529,7 +582,7 @@ export class S3CompatibleStorage implements StorageAdapter {
// Convert connections Map to a serializable format
const serializableEdge = {
...edge,
connections: this.mapToObject(edge.connections, (set) => Array.from(set))
connections: this.mapToObject(edge.connections, (set) => Array.from(set as Set<string>))
}
// Import the PutObjectCommand only when needed