soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: add Firestore vector search integration and noun type-based search enhancements

Browse source 
Introduced Firestore vector search integration using `@firebase/firestore-vector-search`. Added support for restricting searches by specific noun types to optimize performance and relevance. Updated `README.md` to document the new functionality and usage examples.
This commit is contained in:
David Snelling 2025-06-02 12:23:11 -07:00
parent b10a32e8de
commit 725610fbe4
13 changed files with 2439 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

480
src/augmentations/firestoreStorageAugmentation.ts Normal file
View file

@ -0,0 +1,480 @@
import {
AugmentationType,
IMemoryAugmentation,
AugmentationResponse
} from '../types/augmentations.js'
import { Vector } from '../coreTypes.js'
import { cosineDistance } from '../utils/distance.js'
import { initializeApp, getApp } from 'firebase/app'
import {
getFirestore,
collection,
doc,
setDoc,
getDoc,
updateDoc,
deleteDoc,
query,
getDocs,
limit
} from 'firebase/firestore'
import { findNearest } from '@firebase/firestore-vector-search'
/**
* Configuration for Firestore storage augmentation
*/
export interface FirestoreStorageConfig {
/**
* Firestore project ID
*/
projectId: string
/**
* Firestore collection name for storing data
*/
collection: string
/**
* Optional Firestore credentials
*/
credentials?: any
/**
* Optional Firestore database URL
*/
databaseURL?: string
/**
* Optional Firestore app name
*/
appName?: string
}
/**
* Storage augmentation that uses Firestore for storage
*/
export class FirestoreStorageAugmentation implements IMemoryAugmentation {
readonly name: string
readonly description: string = 'Storage augmentation that stores data in Firestore'
enabled: boolean = true
private config: FirestoreStorageConfig
private isInitialized = false
private firestore: any = null
private collection: any = null
constructor(name: string, config: FirestoreStorageConfig) {
this.name = name
this.config = config
}
getType(): AugmentationType {
return AugmentationType.MEMORY
}
async initialize(): Promise<void> {
if (this.isInitialized) {
return
}
try {
// Initialize Firebase if not already initialized
let app
try {
app = getApp(this.config.appName || '[DEFAULT]')
} catch (e) {
// App doesn't exist, initialize it
// Create a config object with the required properties
const firebaseConfig: any = {
projectId: this.config.projectId
}
// Add optional properties if they exist
if (this.config.databaseURL) {
firebaseConfig.databaseURL = this.config.databaseURL
}
// Add credentials if they exist
if (this.config.credentials) {
// Use credentials as part of the config object
// instead of as a 'credential' property
Object.assign(firebaseConfig, this.config.credentials)
}
app = initializeApp(firebaseConfig, this.config.appName)
}
// Get Firestore instance
this.firestore = getFirestore(app)
this.collection = collection(this.firestore, this.config.collection)
this.isInitialized = true
} catch (error) {
console.error(`Failed to initialize FirestoreStorageAugmentation:`, error)
throw new Error(`Failed to initialize FirestoreStorageAugmentation: ${error}`)
}
}
async shutDown(): Promise<void> {
this.isInitialized = false
this.firestore = null
this.collection = null
}
async getStatus(): Promise<'active' | 'inactive' | 'error'> {
if (!this.isInitialized) {
return 'inactive'
}
try {
// Try a simple operation to check if Firestore is working
await getDocs(query(collection(this.firestore, '__test__'), limit(1)))
return 'active'
} catch (error) {
console.error('Firestore connection error:', error)
return 'error'
}
}
async storeData(
key: string,
data: unknown,
options?: Record<string, unknown>
): Promise<AugmentationResponse<boolean>> {
await this.ensureInitialized()
try {
await setDoc(doc(this.collection, key), this.prepareForFirestore(data))
return { success: true, data: true }
} catch (error) {
console.error(`Failed to store data for key ${key}:`, error)
return {
success: false,
data: false,
error: `Failed to store data: ${error}`
}
}
}
async retrieveData(
key: string,
options?: Record<string, unknown>
): Promise<AugmentationResponse<unknown>> {
await this.ensureInitialized()
try {
const docSnapshot = await getDoc(doc(this.collection, key))
if (!docSnapshot.exists()) {
return {
success: true,
data: null
}
}
return {
success: true,
data: docSnapshot.data()
}
} catch (error) {
console.error(`Failed to retrieve data for key ${key}:`, error)
return {
success: false,
data: null,
error: `Failed to retrieve data: ${error}`
}
}
}
async updateData(
key: string,
data: unknown,
options?: Record<string, unknown>
): Promise<AugmentationResponse<boolean>> {
await this.ensureInitialized()
try {
await updateDoc(doc(this.collection, key), this.prepareForFirestore(data))
return { success: true, data: true }
} catch (error) {
console.error(`Failed to update data for key ${key}:`, error)
return {
success: false,
data: false,
error: `Failed to update data: ${error}`
}
}
}
async deleteData(
key: string,
options?: Record<string, unknown>
): Promise<AugmentationResponse<boolean>> {
await this.ensureInitialized()
try {
await deleteDoc(doc(this.collection, key))
return { success: true, data: true }
} catch (error) {
console.error(`Failed to delete data for key ${key}:`, error)
return {
success: false,
data: false,
error: `Failed to delete data: ${error}`
}
}
}
async listDataKeys(
pattern?: string,
options?: Record<string, unknown>
): Promise<AugmentationResponse<string[]>> {
await this.ensureInitialized()
try {
// Create a query from the collection
const q = query(this.collection)
// If pattern is provided, use it to filter keys
// Note: Firestore doesn't support wildcard queries directly,
// so we'll need to do some client-side filtering
const snapshot = await getDocs(q)
let keys = snapshot.docs.map((docSnapshot: any) => docSnapshot.id)
// Apply pattern filtering if provided
if (pattern) {
// Convert wildcard pattern to regex
const regexPattern = new RegExp(
'^' + pattern.replace(/\*/g, '.*') + '$'
)
keys = keys.filter((key: string) => regexPattern.test(key))
}
return {
success: true,
data: keys
}
} catch (error) {
console.error(`Failed to list data keys:`, error)
return {
success: false,
data: [],
error: `Failed to list data keys: ${error}`
}
}
}
/**
* Searches for data in Firestore using vector similarity.
* Uses Firestore's built-in findNearest function for efficient vector search.
* @param queryData The query vector or data to search for
* @param k Number of results to return (default: 10)
* @param options Optional search options
*/
async search(
queryData: unknown,
k: number = 10,
options?: Record<string, unknown>
): Promise<AugmentationResponse<Array<{
id: string;
score: number;
data: unknown;
}>>> {
await this.ensureInitialized()
try {
// Check if queryData is a vector
let queryVector: Vector
if (Array.isArray(queryData) && queryData.every(item => typeof item === 'number')) {
queryVector = queryData as Vector
} else {
// If queryData is not a vector, we can't perform vector search
return {
success: false,
data: [],
error: 'Query must be a vector (array of numbers) for vector search'
}
}
// Get vector field name from options or use default 'vector'
const vectorField = options?.vectorField as string || 'vector'
// Get distance measure from options or use default 'COSINE'
// Firestore supports 'COSINE', 'EUCLIDEAN', and 'DOT_PRODUCT'
const distanceMeasure = options?.distanceMeasure as string || 'COSINE'
try {
// Note: In Firebase v9+, vector search requires the Firebase Extensions for Firestore Vector Search
// This code attempts to use it if available, but will fall back to client-side search
// Use the vector search extension imported at the top of the file
try {
const vectorSearchOptions = {
collection: this.collection,
vectorField: vectorField,
queryVector: queryVector,
limit: k,
distanceMeasure: distanceMeasure
}
const searchResults = await findNearest(vectorSearchOptions)
// Process results
const results: Array<{
id: string;
score: number;
data: unknown;
}> = searchResults.map((result: any) => {
// Calculate the similarity score based on the distance measure
let score: number
if (distanceMeasure === 'DOT_PRODUCT') {
// For dot product, higher is already better
score = result.distance || 0
} else {
// For COSINE and EUCLIDEAN, convert to similarity score
// where 1 is perfect match and 0 is completely dissimilar
score = 1 / (1 + (result.distance || 0))
}
return {
id: result.id,
score,
data: result.data
}
})
return {
success: true,
data: results
}
} catch (vectorSearchError) {
// Vector search extension not available, fall back to client-side search
console.warn('Firestore vector search extension not available, falling back to client-side search:', vectorSearchError)
throw vectorSearchError
}
} catch (vectorSearchError) {
console.warn('Firestore vector search failed, falling back to client-side search:', vectorSearchError)
// Fallback to client-side search if findNearest is not available
// This can happen if the Firestore instance doesn't support vector search
// or if the collection isn't configured for vector search
// Get all documents from Firestore using the modern API
const q = query(this.collection)
const snapshot = await getDocs(q)
// Calculate distances and prepare results
const results: Array<{
id: string;
score: number;
data: unknown;
}> = []
for (const docSnapshot of snapshot.docs) {
const data = docSnapshot.data() as Record<string, any>
// Skip documents that don't have a vector field
if (!data[vectorField] || !Array.isArray(data[vectorField])) {
continue
}
// Calculate distance between query vector and document vector
const distance = cosineDistance(queryVector, data[vectorField] as number[])
// Convert distance to similarity score (1 - distance for cosine)
// This way higher scores are better (more similar)
const score = 1 - distance
results.push({
id: docSnapshot.id,
score,
data
})
}
// Sort results by score (descending) and take top k
results.sort((a, b) => b.score - a.score)
const topResults = results.slice(0, k)
return {
success: true,
data: topResults
}
}
} catch (error) {
console.error(`Failed to search in Firestore:`, error)
return {
success: false,
data: [],
error: `Failed to search in Firestore: ${error}`
}
}
}
private async ensureInitialized(): Promise<void> {
if (!this.isInitialized) {
await this.initialize()
}
}
/**
* Prepare data for Firestore by handling special types
*/
private prepareForFirestore(obj: any): any {
if (obj === null || obj === undefined) {
return null
}
if (
typeof obj === 'string' ||
typeof obj === 'number' ||
typeof obj === 'boolean'
) {
return obj
}
if (obj instanceof Date) {
return obj
}
if (Array.isArray(obj)) {
return obj.map(item => this.prepareForFirestore(item))
}
if (obj instanceof Map) {
const result: Record<string, any> = {}
for (const [key, value] of obj.entries()) {
result[String(key)] = this.prepareForFirestore(value)
}
return result
}
if (obj instanceof Set) {
return Array.from(obj).map(item => this.prepareForFirestore(item))
}
if (typeof obj === 'object') {
const result: Record<string, any> = {}
for (const key in obj) {
if (Object.prototype.hasOwnProperty.call(obj, key)) {
result[key] = this.prepareForFirestore(obj[key])
}
}
return result
}
// Default fallback
return null
}
}
/**
* Create a Firestore storage augmentation
*/
export function createFirestoreStorageAugmentation(
name: string,
config: FirestoreStorageConfig
): IMemoryAugmentation {
return new FirestoreStorageAugmentation(name, config)
}

341
src/augmentations/memoryAugmentations.ts Normal file
View file

@ -0,0 +1,341 @@
import {
AugmentationType,
IMemoryAugmentation,
AugmentationResponse
} from '../types/augmentations.js'
import { StorageAdapter, Vector } from '../coreTypes.js'
import { MemoryStorage } from '../storage/opfsStorage.js'
import { FileSystemStorage } from '../storage/fileSystemStorage.js'
import { OPFSStorage } from '../storage/opfsStorage.js'
import { cosineDistance } from '../utils/distance.js'
import {
FirestoreStorageAugmentation,
FirestoreStorageConfig,
createFirestoreStorageAugmentation
} from './firestoreStorageAugmentation.js'
/**
* Base class for memory augmentations that wrap a StorageAdapter
*/
abstract class BaseMemoryAugmentation implements IMemoryAugmentation {
readonly name: string
readonly description: string = 'Base memory augmentation'
enabled: boolean = true
protected storage: StorageAdapter
protected isInitialized = false
constructor(name: string, storage: StorageAdapter) {
this.name = name
this.storage = storage
}
async initialize(): Promise<void> {
if (this.isInitialized) {
return
}
try {
await this.storage.init()
this.isInitialized = true
} catch (error) {
console.error(`Failed to initialize ${this.name}:`, error)
throw new Error(`Failed to initialize ${this.name}: ${error}`)
}
}
async shutDown(): Promise<void> {
this.isInitialized = false
}
async getStatus(): Promise<'active' | 'inactive' | 'error'> {
return this.isInitialized ? 'active' : 'inactive'
}
async storeData(
key: string,
data: unknown,
options?: Record<string, unknown>
): Promise<AugmentationResponse<boolean>> {
await this.ensureInitialized()
try {
await this.storage.saveMetadata(key, data)
return { success: true, data: true }
} catch (error) {
console.error(`Failed to store data for key ${key}:`, error)
return {
success: false,
data: false,
error: `Failed to store data: ${error}`
}
}
}
async retrieveData(
key: string,
options?: Record<string, unknown>
): Promise<AugmentationResponse<unknown>> {
await this.ensureInitialized()
try {
const data = await this.storage.getMetadata(key)
return {
success: true,
data
}
} catch (error) {
console.error(`Failed to retrieve data for key ${key}:`, error)
return {
success: false,
data: null,
error: `Failed to retrieve data: ${error}`
}
}
}
async updateData(
key: string,
data: unknown,
options?: Record<string, unknown>
): Promise<AugmentationResponse<boolean>> {
await this.ensureInitialized()
try {
await this.storage.saveMetadata(key, data)
return { success: true, data: true }
} catch (error) {
console.error(`Failed to update data for key ${key}:`, error)
return {
success: false,
data: false,
error: `Failed to update data: ${error}`
}
}
}
async deleteData(
key: string,
options?: Record<string, unknown>
): Promise<AugmentationResponse<boolean>> {
await this.ensureInitialized()
try {
// There's no direct deleteMetadata method, so we save null
await this.storage.saveMetadata(key, null)
return { success: true, data: true }
} catch (error) {
console.error(`Failed to delete data for key ${key}:`, error)
return {
success: false,
data: false,
error: `Failed to delete data: ${error}`
}
}
}
async listDataKeys(
pattern?: string,
options?: Record<string, unknown>
): Promise<AugmentationResponse<string[]>> {
// This is a limitation of the current StorageAdapter interface
// It doesn't provide a way to list all metadata keys
// We could implement this in the future by extending the StorageAdapter interface
return {
success: false,
data: [],
error: 'listDataKeys is not supported by this storage adapter'
}
}
/**
* Searches for data in the storage using vector similarity.
* Implements the findNearest functionality by calculating distances client-side.
* @param query The query vector or data to search for
* @param k Number of results to return (default: 10)
* @param options Optional search options
*/
async search(
query: unknown,
k: number = 10,
options?: Record<string, unknown>
): Promise<AugmentationResponse<Array<{
id: string;
score: number;
data: unknown;
}>>> {
await this.ensureInitialized()
try {
// Check if query is a vector
let queryVector: Vector
if (Array.isArray(query) && query.every(item => typeof item === 'number')) {
queryVector = query as Vector
} else {
// If query is not a vector, we can't perform vector search
return {
success: false,
data: [],
error: 'Query must be a vector (array of numbers) for vector search'
}
}
// Get all nodes from storage
const nodes = await this.storage.getAllNodes()
// Calculate distances and prepare results
const results: Array<{
id: string;
score: number;
data: unknown;
}> = []
for (const node of nodes) {
// Skip nodes that don't have a vector
if (!node.vector || !Array.isArray(node.vector)) {
continue
}
// Get metadata for the node
const metadata = await this.storage.getMetadata(node.id)
// Calculate distance between query vector and node vector
const distance = cosineDistance(queryVector, node.vector)
// Convert distance to similarity score (1 - distance for cosine)
// This way higher scores are better (more similar)
const score = 1 - distance
results.push({
id: node.id,
score,
data: metadata
})
}
// Sort results by score (descending) and take top k
results.sort((a, b) => b.score - a.score)
const topResults = results.slice(0, k)
return {
success: true,
data: topResults
}
} catch (error) {
console.error(`Failed to search in storage:`, error)
return {
success: false,
data: [],
error: `Failed to search in storage: ${error}`
}
}
}
protected async ensureInitialized(): Promise<void> {
if (!this.isInitialized) {
await this.initialize()
}
}
}
/**
* Memory augmentation that uses in-memory storage
*/
export class MemoryStorageAugmentation extends BaseMemoryAugmentation {
readonly description = 'Memory augmentation that stores data in memory'
enabled = true
constructor(name: string) {
super(name, new MemoryStorage())
}
getType(): AugmentationType {
return AugmentationType.MEMORY
}
}
/**
* Memory augmentation that uses file system storage
*/
export class FileSystemStorageAugmentation extends BaseMemoryAugmentation {
readonly description = 'Memory augmentation that stores data in the file system'
enabled = true
constructor(name: string, rootDirectory?: string) {
super(name, new FileSystemStorage(rootDirectory))
}
getType(): AugmentationType {
return AugmentationType.MEMORY
}
}
/**
* Memory augmentation that uses OPFS (Origin Private File System) storage
*/
export class OPFSStorageAugmentation extends BaseMemoryAugmentation {
readonly description = 'Memory augmentation that stores data in the Origin Private File System'
enabled = true
constructor(name: string) {
super(name, new OPFSStorage())
}
getType(): AugmentationType {
return AugmentationType.MEMORY
}
}
/**
* Factory function to create the appropriate memory augmentation based on the environment
*/
export async function createMemoryAugmentation(
name: string,
options: {
storageType?: 'memory' | 'filesystem' | 'opfs' | 'firestore'
rootDirectory?: string
requestPersistentStorage?: boolean
firestoreConfig?: FirestoreStorageConfig
} = {}
): Promise<IMemoryAugmentation> {
// If a specific storage type is requested, use that
if (options.storageType) {
switch (options.storageType) {
case 'memory':
return new MemoryStorageAugmentation(name)
case 'filesystem':
return new FileSystemStorageAugmentation(name, options.rootDirectory)
case 'opfs':
return new OPFSStorageAugmentation(name)
case 'firestore':
if (!options.firestoreConfig) {
throw new Error('firestoreConfig is required when using Firestore storage')
}
return createFirestoreStorageAugmentation(name, options.firestoreConfig)
}
}
// Otherwise, select based on environment
const isNode = typeof process !== 'undefined' &&
process.versions != null &&
process.versions.node != null
if (isNode) {
// In Node.js, use FileSystemStorage
return new FileSystemStorageAugmentation(name, options.rootDirectory)
} else {
// In browser, try OPFS first
const opfsStorage = new OPFSStorage()
if (opfsStorage.isOPFSAvailable()) {
// Request persistent storage if specified
if (options.requestPersistentStorage) {
await opfsStorage.requestPersistentStorage()
}
return new OPFSStorageAugmentation(name)
} else {
// Fall back to memory storage
return new MemoryStorageAugmentation(name)
}
}
}

119
src/brainyData.ts
View file

@ -216,15 +216,17 @@ export class BrainyData<T = any> {
}
/**
* Search for similar vectors
* Search for similar vectors within specific noun types
* @param queryVectorOrData Query vector or data to search for
* @param k Number of results to return
* @param nounTypes Array of noun types to search within, or null to search all
* @param options Additional options
* @returns Array of search results
*/
public async search(
public async searchByNounTypes(
queryVectorOrData: Vector | any,
k: number = 10,
nounTypes: string[] | null = null,
options: {
forceEmbed?: boolean // Force using the embedding function even if input is a vector
} = {}
@ -256,35 +258,110 @@ export class BrainyData<T = any> {
throw new Error('Query vector is undefined or null')
}
// Search in the index
const results = this.index.search(queryVector, k)
// If no noun types specified, search all nodes
if (!nounTypes || nounTypes.length === 0) {
// Search in the index
const results = this.index.search(queryVector, k)
// Get metadata for each result
const searchResults: SearchResult<T>[] = []
// Get metadata for each result
const searchResults: SearchResult<T>[] = []
for (const [id, score] of results) {
const node = this.index.getNodes().get(id)
if (!node) {
continue
for (const [id, score] of results) {
const node = this.index.getNodes().get(id)
if (!node) {
continue
}
const metadata = await this.storage!.getMetadata(id)
searchResults.push({
id,
score,
vector: node.vector,
metadata
})
}
const metadata = await this.storage!.getMetadata(id)
return searchResults
} else {
// Get nodes for each noun type in parallel
const nodePromises = nounTypes.map(nounType => this.storage!.getNodesByNounType(nounType))
const nodeArrays = await Promise.all(nodePromises)
searchResults.push({
id,
score,
vector: node.vector,
metadata
})
// Combine all nodes
const nodes: HNSWNode[] = []
for (const nodeArray of nodeArrays) {
nodes.push(...nodeArray)
}
// Calculate distances for each node
const results: Array<[string, number]> = []
for (const node of nodes) {
const distance = this.index.getDistanceFunction()(queryVector, node.vector)
results.push([node.id, distance])
}
// Sort by distance (ascending)
results.sort((a, b) => a[1] - b[1])
// Take top k results
const topResults = results.slice(0, k)
// Get metadata for each result
const searchResults: SearchResult<T>[] = []
for (const [id, score] of topResults) {
const node = nodes.find(n => n.id === id)
if (!node) {
continue
}
const metadata = await this.storage!.getMetadata(id)
searchResults.push({
id,
score,
vector: node.vector,
metadata
})
}
return searchResults
}
return searchResults
} catch (error) {
console.error('Failed to search vectors:', error)
throw new Error(`Failed to search vectors: ${error}`)
console.error('Failed to search vectors by noun types:', error)
throw new Error(`Failed to search vectors by noun types: ${error}`)
}
}
/**
* Search for similar vectors
* @param queryVectorOrData Query vector or data to search for
* @param k Number of results to return
* @param options Additional options
* @returns Array of search results
*/
public async search(
queryVectorOrData: Vector | any,
k: number = 10,
options: {
forceEmbed?: boolean, // Force using the embedding function even if input is a vector
nounTypes?: string[] // Optional array of noun types to search within
} = {}
): Promise<SearchResult<T>[]> {
// If noun types are specified, use searchByNounTypes
if (options.nounTypes && options.nounTypes.length > 0) {
return this.searchByNounTypes(queryVectorOrData, k, options.nounTypes, {
forceEmbed: options.forceEmbed
})
}
// Otherwise, search all nodes
return this.searchByNounTypes(queryVectorOrData, k, null, {
forceEmbed: options.forceEmbed
})
}
/**
* Get a vector by ID
*/

7
src/coreTypes.ts
View file

@ -99,6 +99,13 @@ export interface StorageAdapter {
getAllNodes(): Promise<HNSWNode[]>;
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
getNodesByNounType(nounType: string): Promise<HNSWNode[]>;
deleteNode(id: string): Promise<void>;
saveEdge(edge: Edge): Promise<void>;

7
src/hnsw/hnswIndex.ts
View file

@ -364,6 +364,13 @@ export class HNSWIndex {
return this.nodes.size
}
/**
* Get the distance function used by the index
*/
public getDistanceFunction(): DistanceFunction {
return this.distanceFunction
}
/**
* Search within a specific layer
* Returns a map of node IDs to distances, sorted by distance

24
src/index.ts
View file

@ -111,17 +111,27 @@ export type {
// Export augmentation implementations
import {
FirestoreSyncAugmentation,
createFirestoreSyncAugmentation
} from './augmentations/firestoreSyncAugmentation.js'
import type { FirestoreSyncConfig } from './augmentations/firestoreSyncAugmentation.js'
MemoryStorageAugmentation,
FileSystemStorageAugmentation,
OPFSStorageAugmentation,
createMemoryAugmentation
} from './augmentations/memoryAugmentations.js'
import {
FirestoreStorageAugmentation,
createFirestoreStorageAugmentation
} from './augmentations/firestoreStorageAugmentation.js'
import type { FirestoreStorageConfig } from './augmentations/firestoreStorageAugmentation.js'
export {
FirestoreSyncAugmentation,
createFirestoreSyncAugmentation
MemoryStorageAugmentation,
FileSystemStorageAugmentation,
OPFSStorageAugmentation,
FirestoreStorageAugmentation,
createMemoryAugmentation,
createFirestoreStorageAugmentation
}
export type {
FirestoreSyncConfig
FirestoreStorageConfig
}

387
src/storage/fileSystemStorage.ts
View file

@ -10,6 +10,15 @@ const NODES_DIR = 'nodes'
const EDGES_DIR = 'edges'
const METADATA_DIR = 'metadata'
// Constants for noun type directories
const PERSON_DIR = 'person'
const PLACE_DIR = 'place'
const THING_DIR = 'thing'
const EVENT_DIR = 'event'
const CONCEPT_DIR = 'concept'
const CONTENT_DIR = 'content'
const DEFAULT_DIR = 'default' // For nodes without a noun type
/**
* File system storage adapter for Node.js environments
*/
@ -18,6 +27,13 @@ export class FileSystemStorage implements StorageAdapter {
private nodesDir: string
private edgesDir: string
private metadataDir: string
private personDir: string
private placeDir: string
private thingDir: string
private eventDir: string
private conceptDir: string
private contentDir: string
private defaultDir: string
private isInitialized = false
constructor(rootDirectory?: string) {
@ -26,6 +42,13 @@ export class FileSystemStorage implements StorageAdapter {
this.nodesDir = ''
this.edgesDir = ''
this.metadataDir = ''
this.personDir = ''
this.placeDir = ''
this.thingDir = ''
this.eventDir = ''
this.conceptDir = ''
this.contentDir = ''
this.defaultDir = ''
}
/**
@ -53,6 +76,15 @@ export class FileSystemStorage implements StorageAdapter {
this.nodesDir = path.join(this.rootDir, NODES_DIR)
this.edgesDir = path.join(this.rootDir, EDGES_DIR)
this.metadataDir = path.join(this.rootDir, METADATA_DIR)
// Set up noun type directory paths
this.personDir = path.join(this.nodesDir, PERSON_DIR)
this.placeDir = path.join(this.nodesDir, PLACE_DIR)
this.thingDir = path.join(this.nodesDir, THING_DIR)
this.eventDir = path.join(this.nodesDir, EVENT_DIR)
this.conceptDir = path.join(this.nodesDir, CONCEPT_DIR)
this.contentDir = path.join(this.nodesDir, CONTENT_DIR)
this.defaultDir = path.join(this.nodesDir, DEFAULT_DIR)
} catch (importError) {
throw new Error(`Failed to import Node.js modules: ${importError}. This adapter requires a Node.js environment.`)
}
@ -63,6 +95,15 @@ export class FileSystemStorage implements StorageAdapter {
await this.ensureDirectoryExists(this.edgesDir)
await this.ensureDirectoryExists(this.metadataDir)
// Create noun type directories
await this.ensureDirectoryExists(this.personDir)
await this.ensureDirectoryExists(this.placeDir)
await this.ensureDirectoryExists(this.thingDir)
await this.ensureDirectoryExists(this.eventDir)
await this.ensureDirectoryExists(this.conceptDir)
await this.ensureDirectoryExists(this.contentDir)
await this.ensureDirectoryExists(this.defaultDir)
this.isInitialized = true
} catch (error) {
console.error('Failed to initialize file system storage:', error)
@ -83,7 +124,10 @@ export class FileSystemStorage implements StorageAdapter {
connections: this.mapToObject(node.connections, (set) => Array.from(set))
}
const filePath = path.join(this.nodesDir, `${node.id}.json`)
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(node.id)
const filePath = path.join(nodeDir, `${node.id}.json`)
await fs.promises.writeFile(
filePath,
JSON.stringify(serializableNode, null, 2),
@ -102,12 +146,75 @@ export class FileSystemStorage implements StorageAdapter {
await this.ensureInitialized()
try {
const filePath = path.join(this.nodesDir, `${id}.json`)
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(id)
const filePath = path.join(nodeDir, `${id}.json`)
// Check if a file exists
try {
await fs.promises.access(filePath)
} catch {
// If the file doesn't exist in the expected directory, try the default directory
if (nodeDir !== this.defaultDir) {
const defaultFilePath = path.join(this.defaultDir, `${id}.json`)
try {
await fs.promises.access(defaultFilePath)
// If found in default directory, use that path
const data = await fs.promises.readFile(defaultFilePath, 'utf8')
const parsedNode = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
}
} catch {
// If not found in default directory either, try all noun type directories
const directories = [
this.personDir,
this.placeDir,
this.thingDir,
this.eventDir,
this.conceptDir,
this.contentDir
]
for (const dir of directories) {
if (dir === nodeDir) continue // Skip the already checked directory
const dirFilePath = path.join(dir, `${id}.json`)
try {
await fs.promises.access(dirFilePath)
// If found in this directory, use that path
const data = await fs.promises.readFile(dirFilePath, 'utf8')
const parsedNode = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
}
} catch {
// Continue to the next directory
}
}
return null // File doesn't exist in any directory
}
}
return null // File doesn't exist
}
@ -131,6 +238,65 @@ export class FileSystemStorage implements StorageAdapter {
}
}
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
await this.ensureInitialized()
try {
// Determine the directory based on the noun type
let dir: string
switch (nounType) {
case 'person':
dir = this.personDir
break
case 'place':
dir = this.placeDir
break
case 'thing':
dir = this.thingDir
break
case 'event':
dir = this.eventDir
break
case 'concept':
dir = this.conceptDir
break
case 'content':
dir = this.contentDir
break
default:
dir = this.defaultDir
}
const nodes: HNSWNode[] = []
try {
const files = await fs.promises.readdir(dir)
const nodePromises = files
.filter((file: string) => file.endsWith('.json'))
.map((file: string) => {
// Use the file path directly instead of getNode to avoid redundant searches
return this.readNodeFromFile(path.join(dir, file))
})
const dirNodes = await Promise.all(nodePromises)
nodes.push(...dirNodes.filter((node): node is HNSWNode => node !== null))
} catch (dirError) {
// If directory doesn't exist or can't be read, log a warning
console.warn(`Could not read directory for noun type ${nounType}:`, dirError)
}
return nodes
} catch (error) {
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
}
}
/**
* Get all nodes from storage
*/
@ -138,22 +304,59 @@ export class FileSystemStorage implements StorageAdapter {
await this.ensureInitialized()
try {
const files = await fs.promises.readdir(this.nodesDir)
const nodePromises = files
.filter((file: string) => file.endsWith('.json'))
.map((file: string) => {
const id = path.basename(file, '.json')
return this.getNode(id)
})
// Get all noun types
const nounTypes = [
'person',
'place',
'thing',
'event',
'concept',
'content',
'default'
]
const nodes = await Promise.all(nodePromises)
return nodes.filter((node): node is HNSWNode => node !== null)
// Run searches in parallel for all noun types
const nodePromises = nounTypes.map(nounType => this.getNodesByNounType(nounType))
const nodeArrays = await Promise.all(nodePromises)
// Combine all results
const allNodes: HNSWNode[] = []
for (const nodes of nodeArrays) {
allNodes.push(...nodes)
}
return allNodes
} catch (error) {
console.error('Failed to get all nodes:', error)
throw new Error(`Failed to get all nodes: ${error}`)
}
}
/**
* Read a node from a file
*/
private async readNodeFromFile(filePath: string): Promise<HNSWNode | null> {
try {
const data = await fs.promises.readFile(filePath, 'utf8')
const parsedNode = JSON.parse(data)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(parsedNode.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: parsedNode.id,
vector: parsedNode.vector,
connections
}
} catch (error) {
console.error(`Failed to read node from file ${filePath}:`, error)
return null
}
}
/**
* Delete a node from storage
*/
@ -161,16 +364,53 @@ export class FileSystemStorage implements StorageAdapter {
await this.ensureInitialized()
try {
const filePath = path.join(this.nodesDir, `${id}.json`)
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(id)
const filePath = path.join(nodeDir, `${id}.json`)
// Check if a file exists before attempting to delete
try {
await fs.promises.access(filePath)
await fs.promises.unlink(filePath)
return // File found and deleted
} catch {
// If the file doesn't exist in the expected directory, try the default directory
if (nodeDir !== this.defaultDir) {
const defaultFilePath = path.join(this.defaultDir, `${id}.json`)
try {
await fs.promises.access(defaultFilePath)
await fs.promises.unlink(defaultFilePath)
return // File found and deleted
} catch {
// If not found in default directory either, try all noun type directories
const directories = [
this.personDir,
this.placeDir,
this.thingDir,
this.eventDir,
this.conceptDir,
this.contentDir
]
for (const dir of directories) {
if (dir === nodeDir) continue // Skip the already checked directory
const dirFilePath = path.join(dir, `${id}.json`)
try {
await fs.promises.access(dirFilePath)
await fs.promises.unlink(dirFilePath)
return // File found and deleted
} catch {
// Continue to the next directory
}
}
return // File doesn't exist in any directory, nothing to delete
}
}
return // File doesn't exist, nothing to delete
}
await fs.promises.unlink(filePath)
} catch (error) {
console.error(`Failed to delete node ${id}:`, error)
throw new Error(`Failed to delete node ${id}: ${error}`)
@ -392,6 +632,15 @@ export class FileSystemStorage implements StorageAdapter {
await this.ensureDirectoryExists(this.nodesDir)
await this.ensureDirectoryExists(this.edgesDir)
await this.ensureDirectoryExists(this.metadataDir)
// Create noun type directories
await this.ensureDirectoryExists(this.personDir)
await this.ensureDirectoryExists(this.placeDir)
await this.ensureDirectoryExists(this.thingDir)
await this.ensureDirectoryExists(this.eventDir)
await this.ensureDirectoryExists(this.conceptDir)
await this.ensureDirectoryExists(this.contentDir)
await this.ensureDirectoryExists(this.defaultDir)
} catch (error) {
console.error('Failed to clear storage:', error)
throw new Error(`Failed to clear storage: ${error}`)
@ -438,6 +687,22 @@ export class FileSystemStorage implements StorageAdapter {
}
}
/**
* Count the number of JSON files in a directory
*/
private async countFilesInDirectory(dirPath: string): Promise<number> {
try {
const files = await fs.promises.readdir(dirPath)
return files.filter((file: string) => file.endsWith('.json')).length
} catch (error) {
// If the directory doesn't exist, return 0
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
return 0
}
throw error
}
}
/**
* Convert a Map to a plain object for serialization
*/
@ -452,6 +717,42 @@ export class FileSystemStorage implements StorageAdapter {
return obj
}
/**
* Get the appropriate directory for a node based on its metadata
*/
private async getNodeDirectory(id: string): Promise<string> {
try {
// Try to get the metadata for the node
const metadata = await this.getMetadata(id)
// If metadata exists and has a noun field, use the corresponding directory
if (metadata && metadata.noun) {
switch (metadata.noun) {
case 'person':
return this.personDir
case 'place':
return this.placeDir
case 'thing':
return this.thingDir
case 'event':
return this.eventDir
case 'concept':
return this.conceptDir
case 'content':
return this.contentDir
default:
return this.defaultDir
}
}
// If no metadata or no noun field, use the default directory
return this.defaultDir
} catch (error) {
// If there's an error getting the metadata, use the default directory
return this.defaultDir
}
}
/**
* Get information about storage usage and capacity
*/
@ -494,11 +795,67 @@ export class FileSystemStorage implements StorageAdapter {
const edgesDirSize = await calculateDirSize(this.edgesDir)
const metadataDirSize = await calculateDirSize(this.metadataDir)
// Calculate sizes of noun type directories
const personDirSize = await calculateDirSize(this.personDir)
const placeDirSize = await calculateDirSize(this.placeDir)
const thingDirSize = await calculateDirSize(this.thingDir)
const eventDirSize = await calculateDirSize(this.eventDir)
const conceptDirSize = await calculateDirSize(this.conceptDir)
const contentDirSize = await calculateDirSize(this.contentDir)
const defaultDirSize = await calculateDirSize(this.defaultDir)
// Note: The noun type directories are subdirectories of the nodes directory,
// so their sizes are already included in nodesDirSize.
// We don't need to add them again to avoid double counting.
totalSize = nodesDirSize + edgesDirSize + metadataDirSize
// Get filesystem information
let quota = null
let details = {}
let details: {
nounTypes?: {
person: { size: number; count: number };
place: { size: number; count: number };
thing: { size: number; count: number };
event: { size: number; count: number };
concept: { size: number; count: number };
content: { size: number; count: number };
default: { size: number; count: number };
};
availableSpace?: number;
totalSpace?: number;
freePercentage?: number;
} = {
nounTypes: {
person: {
size: personDirSize,
count: await this.countFilesInDirectory(this.personDir)
},
place: {
size: placeDirSize,
count: await this.countFilesInDirectory(this.placeDir)
},
thing: {
size: thingDirSize,
count: await this.countFilesInDirectory(this.thingDir)
},
event: {
size: eventDirSize,
count: await this.countFilesInDirectory(this.eventDir)
},
concept: {
size: conceptDirSize,
count: await this.countFilesInDirectory(this.conceptDir)
},
content: {
size: contentDirSize,
count: await this.countFilesInDirectory(this.contentDir)
},
default: {
size: defaultDirSize,
count: await this.countFilesInDirectory(this.defaultDir)
}
}
}
try {
// Try to get disk space information

660
src/storage/opfsStorage.ts
View file

@ -12,11 +12,27 @@ const EDGES_DIR = 'edges'
const METADATA_DIR = 'metadata'
const DB_INFO_FILE = 'db-info.json'
// Constants for noun type directories
const PERSON_DIR = 'person'
const PLACE_DIR = 'place'
const THING_DIR = 'thing'
const EVENT_DIR = 'event'
const CONCEPT_DIR = 'concept'
const CONTENT_DIR = 'content'
const DEFAULT_DIR = 'default' // For nodes without a noun type
export class OPFSStorage implements StorageAdapter {
private rootDir: FileSystemDirectoryHandle | null = null
private nodesDir: FileSystemDirectoryHandle | null = null
private edgesDir: FileSystemDirectoryHandle | null = null
private metadataDir: FileSystemDirectoryHandle | null = null
private personDir: FileSystemDirectoryHandle | null = null
private placeDir: FileSystemDirectoryHandle | null = null
private thingDir: FileSystemDirectoryHandle | null = null
private eventDir: FileSystemDirectoryHandle | null = null
private conceptDir: FileSystemDirectoryHandle | null = null
private contentDir: FileSystemDirectoryHandle | null = null
private defaultDir: FileSystemDirectoryHandle | null = null
private isInitialized = false
private isAvailable = false
private isPersistentRequested = false
@ -66,6 +82,29 @@ export class OPFSStorage implements StorageAdapter {
create: true
})
// Create or get noun type directories
this.personDir = await this.nodesDir.getDirectoryHandle(PERSON_DIR, {
create: true
})
this.placeDir = await this.nodesDir.getDirectoryHandle(PLACE_DIR, {
create: true
})
this.thingDir = await this.nodesDir.getDirectoryHandle(THING_DIR, {
create: true
})
this.eventDir = await this.nodesDir.getDirectoryHandle(EVENT_DIR, {
create: true
})
this.conceptDir = await this.nodesDir.getDirectoryHandle(CONCEPT_DIR, {
create: true
})
this.contentDir = await this.nodesDir.getDirectoryHandle(CONTENT_DIR, {
create: true
})
this.defaultDir = await this.nodesDir.getDirectoryHandle(DEFAULT_DIR, {
create: true
})
this.isInitialized = true
} catch (error) {
console.error('Failed to initialize OPFS storage:', error)
@ -140,8 +179,11 @@ export class OPFSStorage implements StorageAdapter {
)
}
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(node.id)
// Create or get the file for this node
const fileHandle = await this.nodesDir!.getFileHandle(node.id, {
const fileHandle = await nodeDir.getFileHandle(node.id, {
create: true
})
@ -162,36 +204,181 @@ export class OPFSStorage implements StorageAdapter {
await this.ensureInitialized()
try {
// Get the file handle for this node
const fileHandle = await this.nodesDir!.getFileHandle(id)
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(id)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
try {
// Get the file handle for this node
const fileHandle = await nodeDir.getFileHandle(id)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
return {
id: data.id,
vector: data.vector,
connections
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections
}
} catch (dirError) {
// If the file doesn't exist in the expected directory, try the default directory
if (nodeDir !== this.defaultDir) {
try {
// Get the file handle from the default directory
const fileHandle = await this.defaultDir!.getFileHandle(id)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections
}
} catch (defaultDirError) {
// If not found in default directory either, try all noun type directories
const directories = [
this.personDir!,
this.placeDir!,
this.thingDir!,
this.eventDir!,
this.conceptDir!,
this.contentDir!
]
for (const dir of directories) {
if (dir === nodeDir) continue // Skip the already checked directory
try {
// Get the file handle from this directory
const fileHandle = await dir.getFileHandle(id)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
return {
id: data.id,
vector: data.vector,
connections
}
} catch {
// Continue to the next directory
}
}
return null // File doesn't exist in any directory
}
}
return null // File doesn't exist
}
} catch (error) {
// If the file doesn't exist, return null
if ((error as any).name === 'NotFoundError') {
return null
}
console.error(`Failed to get node ${id}:`, error)
throw new Error(`Failed to get node ${id}: ${error}`)
}
}
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
await this.ensureInitialized()
try {
const nodes: HNSWNode[] = []
// Determine the directory based on the noun type
let dir: FileSystemDirectoryHandle
switch (nounType) {
case 'person':
dir = this.personDir!
break
case 'place':
dir = this.placeDir!
break
case 'thing':
dir = this.thingDir!
break
case 'event':
dir = this.eventDir!
break
case 'concept':
dir = this.conceptDir!
break
case 'content':
dir = this.contentDir!
break
default:
dir = this.defaultDir!
}
try {
// Get all keys (filenames) in this directory
// @ts-ignore - TypeScript doesn't recognize FileSystemDirectoryHandle.keys() properly
const keys = dir.keys()
// Iterate through all keys and get the corresponding nodes
for await (const name of keys) {
try {
// Get the file handle for this node
const fileHandle = await dir.getFileHandle(name)
// Read the node data from the file
const file = await fileHandle.getFile()
const text = await file.text()
const data = JSON.parse(text)
// Convert serialized connections back to Map<number, Set<string>>
const connections = new Map<number, Set<string>>()
for (const [level, nodeIds] of Object.entries(data.connections)) {
connections.set(Number(level), new Set(nodeIds as string[]))
}
nodes.push({
id: data.id,
vector: data.vector,
connections
})
} catch (nodeError) {
console.warn(`Failed to read node ${name} from directory:`, nodeError)
// Continue to the next node
}
}
} catch (dirError) {
console.warn(`Failed to read directory for noun type ${nounType}:`, dirError)
}
return nodes
} catch (error) {
console.error(`Failed to get nodes for noun type ${nounType}:`, error)
throw new Error(`Failed to get nodes for noun type ${nounType}: ${error}`)
}
}
/**
* Get all nodes from storage
*/
@ -199,21 +386,28 @@ export class OPFSStorage implements StorageAdapter {
await this.ensureInitialized()
try {
const nodes: HNSWNode[] = []
// Get all noun types
const nounTypes = [
'person',
'place',
'thing',
'event',
'concept',
'content',
'default'
]
// Get all keys (filenames) in the nodes directory
// @ts-ignore - TypeScript doesn't recognize FileSystemDirectoryHandle.keys() properly
const keys = this.nodesDir!.keys()
// Run searches in parallel for all noun types
const nodePromises = nounTypes.map(nounType => this.getNodesByNounType(nounType))
const nodeArrays = await Promise.all(nodePromises)
// Iterate through all keys and get the corresponding nodes
for await (const name of keys) {
const node = await this.getNode(name)
if (node) {
nodes.push(node)
}
// Combine all results
const allNodes: HNSWNode[] = []
for (const nodes of nodeArrays) {
allNodes.push(...nodes)
}
return nodes
return allNodes
} catch (error) {
console.error('Failed to get all nodes:', error)
throw new Error(`Failed to get all nodes: ${error}`)
@ -227,13 +421,51 @@ export class OPFSStorage implements StorageAdapter {
await this.ensureInitialized()
try {
await this.nodesDir!.removeEntry(id)
} catch (error) {
// Ignore if the file doesn't exist
if ((error as any).name !== 'NotFoundError') {
console.error(`Failed to delete node ${id}:`, error)
throw new Error(`Failed to delete node ${id}: ${error}`)
// Get the appropriate directory based on the node's metadata
const nodeDir = await this.getNodeDirectory(id)
try {
// Try to delete the node from the appropriate directory
await nodeDir.removeEntry(id)
return // Node deleted successfully
} catch (dirError) {
// If the file doesn't exist in the expected directory, try the default directory
if (nodeDir !== this.defaultDir) {
try {
await this.defaultDir!.removeEntry(id)
return // Node deleted successfully
} catch (defaultDirError) {
// If not found in default directory either, try all noun type directories
const directories = [
this.personDir!,
this.placeDir!,
this.thingDir!,
this.eventDir!,
this.conceptDir!,
this.contentDir!
]
for (const dir of directories) {
if (dir === nodeDir) continue // Skip the already checked directory
try {
await dir.removeEntry(id)
return // Node deleted successfully
} catch {
// Continue to the next directory
}
}
// If we get here, the node wasn't found in any directory
return
}
}
// If the file doesn't exist, that's fine
return
}
} catch (error) {
console.error(`Failed to delete node ${id}:`, error)
throw new Error(`Failed to delete node ${id}: ${error}`)
}
}
@ -459,6 +691,29 @@ export class OPFSStorage implements StorageAdapter {
create: true
})
// Create noun type directories
this.personDir = await this.nodesDir.getDirectoryHandle(PERSON_DIR, {
create: true
})
this.placeDir = await this.nodesDir.getDirectoryHandle(PLACE_DIR, {
create: true
})
this.thingDir = await this.nodesDir.getDirectoryHandle(THING_DIR, {
create: true
})
this.eventDir = await this.nodesDir.getDirectoryHandle(EVENT_DIR, {
create: true
})
this.conceptDir = await this.nodesDir.getDirectoryHandle(CONCEPT_DIR, {
create: true
})
this.contentDir = await this.nodesDir.getDirectoryHandle(CONTENT_DIR, {
create: true
})
this.defaultDir = await this.nodesDir.getDirectoryHandle(DEFAULT_DIR, {
create: true
})
// Delete and recreate the edges directory
await this.rootDir!.removeEntry(EDGES_DIR, { recursive: true })
this.edgesDir = await this.rootDir!.getDirectoryHandle(EDGES_DIR, {
@ -499,6 +754,42 @@ export class OPFSStorage implements StorageAdapter {
return obj
}
/**
* Get the appropriate directory for a node based on its metadata
*/
private async getNodeDirectory(id: string): Promise<FileSystemDirectoryHandle> {
try {
// Try to get the metadata for the node
const metadata = await this.getMetadata(id)
// If metadata exists and has a noun field, use the corresponding directory
if (metadata && metadata.noun) {
switch (metadata.noun) {
case 'person':
return this.personDir!
case 'place':
return this.placeDir!
case 'thing':
return this.thingDir!
case 'event':
return this.eventDir!
case 'concept':
return this.conceptDir!
case 'content':
return this.contentDir!
default:
return this.defaultDir!
}
}
// If no metadata or no noun field, use the default directory
return this.defaultDir!
} catch (error) {
// If there's an error getting the metadata, use the default directory
return this.defaultDir!
}
}
/**
* Get information about storage usage and capacity
*/
@ -533,6 +824,22 @@ export class OPFSStorage implements StorageAdapter {
return size
}
// Helper function to count files in a directory
const countFilesInDirectory = async (dirHandle: FileSystemDirectoryHandle): Promise<number> => {
let count = 0
try {
// @ts-ignore - TypeScript doesn't recognize FileSystemDirectoryHandle.entries() properly
for await (const [name, handle] of dirHandle.entries()) {
if (handle.kind === 'file') {
count++
}
}
} catch (error) {
console.warn(`Error counting files in directory:`, error)
}
return count
}
// Calculate size for each directory
if (this.nodesDir) {
totalSize += await calculateDirSize(this.nodesDir)
@ -544,10 +851,49 @@ export class OPFSStorage implements StorageAdapter {
totalSize += await calculateDirSize(this.metadataDir)
}
// Calculate sizes of noun type directories
const personDirSize = this.personDir ? await calculateDirSize(this.personDir) : 0
const placeDirSize = this.placeDir ? await calculateDirSize(this.placeDir) : 0
const thingDirSize = this.thingDir ? await calculateDirSize(this.thingDir) : 0
const eventDirSize = this.eventDir ? await calculateDirSize(this.eventDir) : 0
const conceptDirSize = this.conceptDir ? await calculateDirSize(this.conceptDir) : 0
const contentDirSize = this.contentDir ? await calculateDirSize(this.contentDir) : 0
const defaultDirSize = this.defaultDir ? await calculateDirSize(this.defaultDir) : 0
// Get storage quota information using the Storage API
let quota = null
let details: Record<string, any> = {
isPersistent: await this.isPersistent()
isPersistent: await this.isPersistent(),
nounTypes: {
person: {
size: personDirSize,
count: this.personDir ? await countFilesInDirectory(this.personDir) : 0
},
place: {
size: placeDirSize,
count: this.placeDir ? await countFilesInDirectory(this.placeDir) : 0
},
thing: {
size: thingDirSize,
count: this.thingDir ? await countFilesInDirectory(this.thingDir) : 0
},
event: {
size: eventDirSize,
count: this.eventDir ? await countFilesInDirectory(this.eventDir) : 0
},
concept: {
size: conceptDirSize,
count: this.conceptDir ? await countFilesInDirectory(this.conceptDir) : 0
},
content: {
size: contentDirSize,
count: this.contentDir ? await countFilesInDirectory(this.contentDir) : 0
},
default: {
size: defaultDirSize,
count: this.defaultDir ? await countFilesInDirectory(this.defaultDir) : 0
}
}
}
try {
@ -587,14 +933,62 @@ export class OPFSStorage implements StorageAdapter {
* In-memory storage adapter for environments where OPFS is not available
*/
export class MemoryStorage implements StorageAdapter {
private nodes: Map<string, HNSWNode> = new Map()
// Map of noun type to Map of node ID to node
private nodes: Map<string, Map<string, HNSWNode>> = new Map()
private edges: Map<string, Edge> = new Map()
private metadata: Map<string, any> = new Map()
// Initialize maps for each noun type
constructor() {
this.nodes.set(PERSON_DIR, new Map())
this.nodes.set(PLACE_DIR, new Map())
this.nodes.set(THING_DIR, new Map())
this.nodes.set(EVENT_DIR, new Map())
this.nodes.set(CONCEPT_DIR, new Map())
this.nodes.set(CONTENT_DIR, new Map())
this.nodes.set(DEFAULT_DIR, new Map())
}
public async init(): Promise<void> {
// Nothing to initialize for in-memory storage
}
/**
* Get the appropriate node type for a node based on its metadata
*/
private async getNodeType(id: string): Promise<string> {
try {
// Try to get the metadata for the node
const metadata = await this.getMetadata(id)
// If metadata exists and has a noun field, use the corresponding type
if (metadata && metadata.noun) {
switch (metadata.noun) {
case 'person':
return PERSON_DIR
case 'place':
return PLACE_DIR
case 'thing':
return THING_DIR
case 'event':
return EVENT_DIR
case 'concept':
return CONCEPT_DIR
case 'content':
return CONTENT_DIR
default:
return DEFAULT_DIR
}
}
// If no metadata or no noun field, use the default type
return DEFAULT_DIR
} catch (error) {
// If there's an error getting the metadata, use the default type
return DEFAULT_DIR
}
}
public async saveNode(node: HNSWNode): Promise<void> {
// Create a deep copy to avoid reference issues
const nodeCopy: HNSWNode = {
@ -608,13 +1002,50 @@ export class MemoryStorage implements StorageAdapter {
nodeCopy.connections.set(level, new Set(connections))
}
this.nodes.set(node.id, nodeCopy)
// Get the appropriate node type based on the node's metadata
const nodeType = await this.getNodeType(node.id)
// Get the map for this node type
const nodeMap = this.nodes.get(nodeType)!
// Save the node in the appropriate map
nodeMap.set(node.id, nodeCopy)
}
public async getNode(id: string): Promise<HNSWNode | null> {
const node = this.nodes.get(id)
// Get the appropriate node type based on the node's metadata
const nodeType = await this.getNodeType(id)
// Get the map for this node type
const nodeMap = this.nodes.get(nodeType)!
// Try to get the node from the appropriate map
let node = nodeMap.get(id)
// If not found in the expected map, try other maps
if (!node) {
return null
// If the node type is not the default type, try the default map
if (nodeType !== DEFAULT_DIR) {
const defaultMap = this.nodes.get(DEFAULT_DIR)!
node = defaultMap.get(id)
// If still not found, try all other maps
if (!node) {
const nodeTypes = [PERSON_DIR, PLACE_DIR, THING_DIR, EVENT_DIR, CONCEPT_DIR, CONTENT_DIR]
for (const type of nodeTypes) {
if (type === nodeType) continue // Skip the already checked map
const typeMap = this.nodes.get(type)!
node = typeMap.get(id)
if (node) break // Found the node, exit the loop
}
}
}
// If still not found, return null
if (!node) {
return null
}
}
// Return a deep copy to avoid reference issues
@ -632,10 +1063,41 @@ export class MemoryStorage implements StorageAdapter {
return nodeCopy
}
public async getAllNodes(): Promise<HNSWNode[]> {
/**
* Get nodes by noun type
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
public async getNodesByNounType(nounType: string): Promise<HNSWNode[]> {
const nodes: HNSWNode[] = []
for (const nodeId of this.nodes.keys()) {
// Get the map for this noun type
let typeMap: Map<string, HNSWNode>
switch (nounType) {
case 'person':
typeMap = this.nodes.get(PERSON_DIR)!
break
case 'place':
typeMap = this.nodes.get(PLACE_DIR)!
break
case 'thing':
typeMap = this.nodes.get(THING_DIR)!
break
case 'event':
typeMap = this.nodes.get(EVENT_DIR)!
break
case 'concept':
typeMap = this.nodes.get(CONCEPT_DIR)!
break
case 'content':
typeMap = this.nodes.get(CONTENT_DIR)!
break
default:
typeMap = this.nodes.get(DEFAULT_DIR)!
}
// Get all nodes from this map
for (const nodeId of typeMap.keys()) {
const node = await this.getNode(nodeId)
if (node) {
nodes.push(node)
@ -645,8 +1107,61 @@ export class MemoryStorage implements StorageAdapter {
return nodes
}
public async getAllNodes(): Promise<HNSWNode[]> {
// Get all noun types
const nounTypes = [
'person',
'place',
'thing',
'event',
'concept',
'content',
'default'
]
// Run searches in parallel for all noun types
const nodePromises = nounTypes.map(nounType => this.getNodesByNounType(nounType))
const nodeArrays = await Promise.all(nodePromises)
// Combine all results
const allNodes: HNSWNode[] = []
for (const nodes of nodeArrays) {
allNodes.push(...nodes)
}
return allNodes
}
public async deleteNode(id: string): Promise<void> {
this.nodes.delete(id)
// Get the appropriate node type based on the node's metadata
const nodeType = await this.getNodeType(id)
// Get the map for this node type
const nodeMap = this.nodes.get(nodeType)!
// Try to delete the node from the appropriate map
const deleted = nodeMap.delete(id)
// If not found in the expected map, try other maps
if (!deleted) {
// If the node type is not the default type, try the default map
if (nodeType !== DEFAULT_DIR) {
const defaultMap = this.nodes.get(DEFAULT_DIR)!
const deletedFromDefault = defaultMap.delete(id)
// If still not found, try all other maps
if (!deletedFromDefault) {
const nodeTypes = [PERSON_DIR, PLACE_DIR, THING_DIR, EVENT_DIR, CONCEPT_DIR, CONTENT_DIR]
for (const type of nodeTypes) {
if (type === nodeType) continue // Skip the already checked map
const typeMap = this.nodes.get(type)!
const deletedFromType = typeMap.delete(id)
if (deletedFromType) break // Node deleted, exit the loop
}
}
}
}
}
public async saveMetadata(id: string, metadata: any): Promise<void> {
@ -772,7 +1287,13 @@ export class MemoryStorage implements StorageAdapter {
}
public async clear(): Promise<void> {
this.nodes.clear()
// Clear all noun type maps
const nodeTypes = [PERSON_DIR, PLACE_DIR, THING_DIR, EVENT_DIR, CONCEPT_DIR, CONTENT_DIR, DEFAULT_DIR]
for (const type of nodeTypes) {
const typeMap = this.nodes.get(type)!
typeMap.clear()
}
this.edges.clear()
this.metadata.clear()
}
@ -833,27 +1354,56 @@ export class MemoryStorage implements StorageAdapter {
return 0
}
// Estimate size of nodes
for (const node of this.nodes.values()) {
totalSize += estimateSize(node)
// Calculate sizes and counts for each noun type
const nounTypeDetails: Record<string, { size: number; count: number }> = {}
const nodeTypes = [PERSON_DIR, PLACE_DIR, THING_DIR, EVENT_DIR, CONCEPT_DIR, CONTENT_DIR, DEFAULT_DIR]
let totalNodeCount = 0
let nodesSize = 0
for (const type of nodeTypes) {
const typeMap = this.nodes.get(type)!
let typeSize = 0
for (const node of typeMap.values()) {
typeSize += estimateSize(node)
}
nounTypeDetails[type] = {
size: typeSize,
count: typeMap.size
}
nodesSize += typeSize
totalNodeCount += typeMap.size
}
totalSize += nodesSize
// Estimate size of edges
let edgesSize = 0
for (const edge of this.edges.values()) {
totalSize += estimateSize(edge)
edgesSize += estimateSize(edge)
}
totalSize += edgesSize
// Estimate size of metadata
let metadataSize = 0
for (const meta of this.metadata.values()) {
totalSize += estimateSize(meta)
metadataSize += estimateSize(meta)
}
totalSize += metadataSize
// Get memory information if available
let quota = null
let details: Record<string, any> = {
nodesCount: this.nodes.size,
edgesCount: this.edges.size,
metadataCount: this.metadata.size
nodeCount: totalNodeCount,
edgeCount: this.edges.size,
metadataCount: this.metadata.size,
nounTypes: nounTypeDetails,
nodesSize,
edgesSize,
metadataSize
}
// Try to get memory information if in a browser environment

17
src/types/firebase-vector-search.d.ts vendored Normal file
View file

@ -0,0 +1,17 @@
declare module '@firebase/firestore-vector-search' {
interface VectorSearchOptions {
collection: any;
vectorField: string;
queryVector: number[];
limit: number;
distanceMeasure?: string;
}
interface VectorSearchResult {
id: string;
data: any;
distance?: number;
}
export function findNearest(options: VectorSearchOptions): Promise<VectorSearchResult[]>;
}

23
src/types/graphTypes.ts
View file

@ -94,6 +94,15 @@ export interface Concept extends GraphNoun {
noun: typeof NounType.Concept
}
export interface Group extends GraphNoun {
noun: typeof NounType.Group
}
export interface List extends GraphNoun {
noun: typeof NounType.List
}
/**
* Represents content (text, media, etc.) in the graph
*/
@ -105,13 +114,17 @@ export interface Content extends GraphNoun {
* Defines valid noun types for graph entities
* Used for categorizing different types of nodes
*/
export const NounType = {
Person: 'person', // Person entities
Place: 'place', // Physical locations
Thing: 'thing', // Physical or virtual objects
Event: 'event', // Events or occurrences
Concept: 'concept', // Abstract concepts or ideas
Content: 'content' // Content items
Content: 'content', // Content items
Group: 'group', // Groups of related entities
List: 'list', // Ordered collections of entities
Category: 'category' // Categories for content items including tags
} as const
export type NounType = (typeof NounType)[keyof typeof NounType]
@ -125,6 +138,12 @@ export const VerbType = {
Created: 'created', // Indicates creation or authorship
Earned: 'earned', // Indicates achievement or acquisition
Owns: 'owns', // Indicates ownership
MemberOf: 'memberOf' // Indicates membership or affiliation
MemberOf: 'memberOf', // Indicates membership or affiliation
RelatedTo: 'relatedTo', // Indicates family relationship
WorksWith: 'worksWith', // Indicates professional relationship
FriendOf: 'friendOf', // Indicates friendship
ReportsTo: 'reportsTo', // Indicates reporting relationship
Supervises: 'supervises', // Indicates supervisory relationship
Mentors: 'mentors' // Indicates mentorship relationship
} as const
export type VerbType = (typeof VerbType)[keyof typeof VerbType]