soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: correct typo in README major updates section

Browse source 
🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-08-06 12:29:32 -07:00
parent d2ddb9199e
commit 1a4f035ffc
22 changed files with 4423 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

334
src/brainyData.ts
View file

@ -32,6 +32,8 @@ import {
batchEmbed
} from './utils/index.js'
import { getAugmentationVersion } from './utils/version.js'
import { matchesMetadataFilter } from './utils/metadataFilter.js'
import { MetadataIndexManager, MetadataIndexConfig } from './utils/metadataIndex.js'
import { NounType, VerbType, GraphNoun } from './types/graphTypes.js'
import {
ServerSearchConduitAugmentation,
@ -196,6 +198,11 @@ export interface BrainyDataConfig {
verbose?: boolean
}
/**
* Metadata indexing configuration
*/
metadataIndex?: MetadataIndexConfig
/**
* Search result caching configuration
* Improves performance for repeated queries
@ -371,8 +378,9 @@ export interface BrainyDataConfig {
}
export class BrainyData<T = any> implements BrainyDataInterface<T> {
private index: HNSWIndex | HNSWIndexOptimized
public index: HNSWIndex | HNSWIndexOptimized // Made public for testing
private storage: StorageAdapter | null = null
public metadataIndex: MetadataIndexManager | null = null
private isInitialized = false
private isInitializing = false
private embeddingFunction: EmbeddingFunction
@ -383,6 +391,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
private lazyLoadInReadOnlyMode: boolean
private writeOnly: boolean
private storageConfig: BrainyDataConfig['storage'] = {}
private config: BrainyDataConfig
private useOptimizedIndex: boolean = false
private _dimensions: number
private loggingConfig: BrainyDataConfig['logging'] = { verbose: true }
@ -407,6 +416,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
updateIndex: true
}
private updateTimerId: NodeJS.Timeout | null = null
private maintenanceIntervals: NodeJS.Timeout[] = []
private lastUpdateTime = 0
private lastKnownNounCount = 0
@ -453,6 +463,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
* Create a new vector database
*/
constructor(config: BrainyDataConfig = {}) {
// Store config
this.config = config
// Set dimensions to fixed value of 384 (all-MiniLM-L6-v2 dimension)
this._dimensions = 384
@ -466,12 +479,12 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
hnswConfig.useDiskBasedIndex = true
}
this.index = new HNSWIndexOptimized(
// Temporarily use base HNSW index for metadata filtering
this.index = new HNSWIndex(
hnswConfig,
this.distanceFunction,
config.storageAdapter || null
this.distanceFunction
)
this.useOptimizedIndex = true
this.useOptimizedIndex = false
// Set storage if provided, otherwise it will be initialized in init()
this.storage = config.storageAdapter || null
@ -740,6 +753,28 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
}
/**
* Start metadata index maintenance
*/
private startMetadataIndexMaintenance(): void {
if (!this.metadataIndex) return
// Flush index periodically to persist changes
const flushInterval = setInterval(async () => {
try {
await this.metadataIndex!.flush()
} catch (error) {
console.warn('Error flushing metadata index:', error)
}
}, 30000) // Flush every 30 seconds
// Store the interval ID for cleanup
if (!this.maintenanceIntervals) {
this.maintenanceIntervals = []
}
this.maintenanceIntervals.push(flushInterval)
}
/**
* Disable real-time updates
*/
@ -1224,11 +1259,37 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Ignore errors loading existing statistics
}
// Initialize metadata index if not in write-only mode
if (!this.writeOnly) {
this.metadataIndex = new MetadataIndexManager(
this.storage!,
this.config.metadataIndex
)
// Check if we need to rebuild the index (for existing data)
const stats = await this.metadataIndex.getStats()
if (stats.totalEntries === 0 && !this.readOnly) {
if (this.loggingConfig?.verbose) {
console.log('Rebuilding metadata index for existing data...')
}
await this.metadataIndex.rebuild()
if (this.loggingConfig?.verbose) {
const newStats = await this.metadataIndex.getStats()
console.log(`Metadata index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`)
}
}
}
this.isInitialized = true
this.isInitializing = false
// Start real-time updates if enabled
this.startRealtimeUpdates()
// Start metadata index maintenance
if (this.metadataIndex) {
this.startMetadataIndexMaintenance()
}
} catch (error) {
console.error('Failed to initialize BrainyData:', error)
this.isInitializing = false
@ -1654,6 +1715,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
await this.storage!.saveMetadata(id, metadataToSave)
// Update metadata index
if (this.metadataIndex && !this.readOnly && !this.frozen) {
await this.metadataIndex.addToIndex(id, metadataToSave)
}
// Track metadata statistics
const metadataService = this.getServiceName(options)
await this.storage!.incrementStatistic('metadata', metadataService)
@ -1987,6 +2053,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
options: {
forceEmbed?: boolean // Force using the embedding function even if input is a vector
service?: string // Filter results by the service that created the data
metadata?: any // Metadata filter criteria
offset?: number // Number of results to skip for pagination (default: 0)
} = {}
): Promise<SearchResult<T>[]> {
@ -2086,12 +2153,84 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
}
// Create filter function for HNSW search with metadata index optimization
const hasMetadataFilter = options.metadata && Object.keys(options.metadata).length > 0
const hasServiceFilter = !!options.service
let filterFunction: ((id: string) => Promise<boolean>) | undefined
let preFilteredIds: Set<string> | undefined
// Use metadata index for pre-filtering if available
if (hasMetadataFilter && this.metadataIndex) {
try {
// Get candidate IDs from metadata index
const candidateIds = await this.metadataIndex.getIdsForFilter(options.metadata)
if (candidateIds.length > 0) {
preFilteredIds = new Set(candidateIds)
// Create a simple filter function that just checks the pre-filtered set
filterFunction = async (id: string) => {
if (!preFilteredIds!.has(id)) return false
// Still apply service filter if needed
if (hasServiceFilter) {
const metadata = await this.storage!.getMetadata(id)
const noun = this.index.getNouns().get(id)
if (!noun || !metadata) return false
const result = { id, score: 0, vector: noun.vector, metadata }
return this.filterResultsByService([result], options.service).length > 0
}
return true
}
} else {
// No items match the metadata criteria, return empty results immediately
return []
}
} catch (indexError) {
console.warn('Metadata index error, falling back to full filtering:', indexError)
// Fall back to full metadata filtering below
}
}
// Fallback to full metadata filtering if index wasn't used
if (!filterFunction && (hasMetadataFilter || hasServiceFilter)) {
filterFunction = async (id: string) => {
// Get metadata for filtering
let metadata = await this.storage!.getMetadata(id)
if (metadata === null) {
metadata = {} as T
}
// Apply metadata filter
if (hasMetadataFilter) {
const matches = matchesMetadataFilter(metadata, options.metadata)
if (!matches) {
return false
}
}
// Apply service filter
if (hasServiceFilter) {
const noun = this.index.getNouns().get(id)
if (!noun) return false
const result = { id, score: 0, vector: noun.vector, metadata }
if (!this.filterResultsByService([result], options.service).length) {
return false
}
}
return true
}
}
// When using offset, we need to fetch more results and then slice
const offset = options.offset || 0
const totalNeeded = k + offset
// Search in the index for totalNeeded results
const results = await this.index.search(queryVector, totalNeeded)
// Search in the index with filter
const results = await this.index.search(queryVector, totalNeeded, filterFunction)
// Skip the offset number of results
const paginatedResults = results.slice(offset, offset + k)
@ -2125,8 +2264,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
})
}
// Filter results by service if specified
return this.filterResultsByService(searchResults, options.service)
return searchResults
} else {
// Get nouns for each noun type in parallel
const nounPromises = nounTypes.map((nounType) =>
@ -2186,8 +2324,8 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
})
}
// Filter results by service if specified
return this.filterResultsByService(searchResults, options.service)
// Results are already filtered, just return them
return searchResults
}
} catch (error) {
console.error('Failed to search vectors by noun types:', error)
@ -2217,6 +2355,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
service?: string // Filter results by the service that created the data
searchField?: string // Optional specific field to search within JSON documents
filter?: { domain?: string } // Filter results by domain
metadata?: any // Metadata filter - supports both simple object matching and MongoDB-style operators
offset?: number // Number of results to skip for pagination (default: 0)
skipCache?: boolean // Skip cache for this search (default: false)
} = {}
@ -2281,29 +2420,41 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Default behavior (backward compatible): search locally
try {
// Check cache first (transparent to user)
const cacheKey = this.searchCache.getCacheKey(
queryVectorOrData,
k,
options
)
const cachedResults = this.searchCache.get(cacheKey)
const hasMetadataFilter = options.metadata && Object.keys(options.metadata).length > 0
// Check cache first (transparent to user) - but skip cache if we have metadata filters
if (!hasMetadataFilter) {
const cacheKey = this.searchCache.getCacheKey(
queryVectorOrData,
k,
options
)
const cachedResults = this.searchCache.get(cacheKey)
if (cachedResults) {
// Track cache hit in health monitor
if (this.healthMonitor) {
const latency = Date.now() - startTime
this.healthMonitor.recordRequest(latency, false)
this.healthMonitor.recordCacheAccess(true)
if (cachedResults) {
// Track cache hit in health monitor
if (this.healthMonitor) {
const latency = Date.now() - startTime
this.healthMonitor.recordRequest(latency, false)
this.healthMonitor.recordCacheAccess(true)
}
return cachedResults
}
return cachedResults
}
// Cache miss - perform actual search
const results = await this.searchLocal(queryVectorOrData, k, options)
const results = await this.searchLocal(queryVectorOrData, k, {
...options,
metadata: options.metadata
})
// Cache results for future queries (unless explicitly disabled)
if (!options.skipCache) {
// Cache results for future queries (unless explicitly disabled or has metadata filter)
if (!options.skipCache && !hasMetadataFilter) {
const cacheKey = this.searchCache.getCacheKey(
queryVectorOrData,
k,
options
)
this.searchCache.set(cacheKey, results)
}
@ -2419,6 +2570,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
searchField?: string // Optional specific field to search within JSON documents
priorityFields?: string[] // Fields to prioritize when searching JSON documents
filter?: { domain?: string } // Filter results by domain
metadata?: any // Metadata filter criteria
offset?: number // Number of results to skip for pagination (default: 0)
skipCache?: boolean // Skip cache for this search (default: false)
} = {}
@ -2485,6 +2637,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
{
forceEmbed: options.forceEmbed,
service: options.service,
metadata: options.metadata,
offset: options.offset
}
)
@ -2493,6 +2646,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
searchResults = await this.searchByNounTypes(queryToUse, k, null, {
forceEmbed: options.forceEmbed,
service: options.service,
metadata: options.metadata,
offset: options.offset
})
}
@ -2901,6 +3055,14 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Try to remove metadata (ignore errors)
try {
// Get metadata before removing for index cleanup
const existingMetadata = await this.storage!.getMetadata(actualId)
// Remove from metadata index
if (this.metadataIndex && existingMetadata && !this.readOnly && !this.frozen) {
await this.metadataIndex.removeFromIndex(actualId, existingMetadata)
}
await this.storage!.saveMetadata(actualId, null)
await this.storage!.decrementStatistic('metadata', service)
} catch (error) {
@ -3012,6 +3174,20 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Update metadata
await this.storage!.saveMetadata(id, metadata)
// Update metadata index
if (this.metadataIndex && !this.readOnly && !this.frozen) {
// Remove old metadata from index if it exists
const oldMetadata = await this.storage!.getMetadata(id)
if (oldMetadata) {
await this.metadataIndex.removeFromIndex(id, oldMetadata)
}
// Add new metadata to index
if (metadata) {
await this.metadataIndex.addToIndex(id, metadata)
}
}
// Track metadata statistics
const service = this.getServiceName(options)
await this.storage!.incrementStatistic('metadata', service)
@ -3454,6 +3630,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Save the complete verb (BaseStorage will handle the separation)
await this.storage!.saveVerb(fullVerb)
// Update metadata index
if (this.metadataIndex && verbMetadata) {
await this.metadataIndex.addToIndex(id, verbMetadata)
}
// Track verb statistics
const serviceForStats = this.getServiceName(options)
await this.storage!.incrementStatistic('verb', serviceForStats)
@ -3701,12 +3882,20 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.checkReadOnly()
try {
// Get existing metadata before removal for index cleanup
const existingMetadata = await this.storage!.getVerbMetadata(id)
// Remove from index
const removed = this.index.removeItem(id)
if (!removed) {
return false
}
// Remove from metadata index
if (this.metadataIndex && existingMetadata) {
await this.metadataIndex.removeFromIndex(id, existingMetadata)
}
// Remove from storage
await this.storage!.deleteVerb(id)
@ -4736,6 +4925,73 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
}
}
/**
* Search within a specific set of items
* This is useful when you've pre-filtered items and want to search only within them
*
* @param queryVectorOrData Query vector or data to search for
* @param itemIds Array of item IDs to search within
* @param k Number of results to return
* @param options Additional options
* @returns Array of search results
*/
public async searchWithinItems(
queryVectorOrData: Vector | any,
itemIds: string[],
k: number = 10,
options: {
forceEmbed?: boolean
} = {}
): Promise<SearchResult<T>[]> {
await this.ensureInitialized()
// Check if database is in write-only mode
this.checkWriteOnly()
// Create a Set for fast lookups
const allowedIds = new Set(itemIds)
// Create filter function that only allows specified items
const filterFunction = async (id: string) => allowedIds.has(id)
// Get query vector
let queryVector: Vector
if (Array.isArray(queryVectorOrData) && !options.forceEmbed) {
queryVector = queryVectorOrData
} else {
queryVector = await this.embeddingFunction(queryVectorOrData)
}
// Search with the filter
const results = await this.index.search(queryVector, Math.min(k, itemIds.length), filterFunction)
// Get metadata for each result
const searchResults: SearchResult<T>[] = []
for (const [id, score] of results) {
const noun = this.index.getNouns().get(id)
if (!noun) continue
let metadata = await this.storage!.getMetadata(id)
if (metadata === null) {
metadata = {} as T
}
if (metadata && typeof metadata === 'object') {
metadata = { ...metadata, id } as T
}
searchResults.push({
id,
score,
vector: noun.vector,
metadata: metadata as T
})
}
return searchResults
}
/**
* Search for similar documents using a text query
* This is a convenience method that embeds the query text and performs a search
@ -4752,6 +5008,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
nounTypes?: string[]
includeVerbs?: boolean
searchMode?: 'local' | 'remote' | 'combined'
metadata?: any // Simple metadata filter - just pass an object with the fields you want to match
} = {}
): Promise<SearchResult<T>[]> {
await this.ensureInitialized()
@ -4765,11 +5022,13 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
// Embed the query text
const queryVector = await this.embed(query)
// Search using the embedded vector
// Search using the embedded vector with metadata filtering
const results = await this.search(queryVector, k, {
nounTypes: options.nounTypes,
includeVerbs: options.includeVerbs,
searchMode: options.searchMode
searchMode: options.searchMode,
metadata: options.metadata,
forceEmbed: false // Already embedded
})
// Track search performance
@ -5729,6 +5988,21 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
this.updateTimerId = null
}
// Stop maintenance intervals
for (const intervalId of this.maintenanceIntervals) {
clearInterval(intervalId)
}
this.maintenanceIntervals = []
// Flush metadata index one last time
if (this.metadataIndex) {
try {
await this.metadataIndex.flush()
} catch (error) {
console.warn('Error flushing metadata index during cleanup:', error)
}
}
// Clean up distributed mode resources
if (this.healthMonitor) {
this.healthMonitor.stop()

87
src/hnsw/hnswIndex.ts
View file

@ -280,7 +280,8 @@ export class HNSWIndex {
*/
public async search(
queryVector: Vector,
k: number = 10
k: number = 10,
filter?: (id: string) => Promise<boolean>
): Promise<Array<[string, number]>> {
if (this.nouns.size === 0) {
return []
@ -372,11 +373,14 @@ export class HNSWIndex {
}
// Search at level 0 with ef = k
// If we have a filter, increase ef to compensate for filtered results
const ef = filter ? Math.max(this.config.efSearch * 3, k * 3) : Math.max(this.config.efSearch, k)
const nearestNouns = await this.searchLayer(
queryVector,
currObj,
Math.max(this.config.efSearch, k),
0
ef,
0,
filter
)
// Convert to array and sort by distance
@ -599,24 +603,25 @@ export class HNSWIndex {
queryVector: Vector,
entryPoint: HNSWNoun,
ef: number,
level: number
level: number,
filter?: (id: string) => Promise<boolean>
): Promise<Map<string, number>> {
// Set of visited nouns
const visited = new Set<string>([entryPoint.id])
// Check if entry point passes filter
const entryPointDistance = this.distanceFunction(queryVector, entryPoint.vector)
const entryPointPasses = filter ? await filter(entryPoint.id) : true
// Priority queue of candidates (closest first)
const candidates = new Map<string, number>()
candidates.set(
entryPoint.id,
this.distanceFunction(queryVector, entryPoint.vector)
)
candidates.set(entryPoint.id, entryPointDistance)
// Priority queue of nearest neighbors found so far (closest first)
const nearest = new Map<string, number>()
nearest.set(
entryPoint.id,
this.distanceFunction(queryVector, entryPoint.vector)
)
if (entryPointPasses) {
nearest.set(entryPoint.id, entryPointDistance)
}
// While there are candidates to explore
while (candidates.size > 0) {
@ -660,17 +665,25 @@ export class HNSWIndex {
// Process the results
for (const { id, distance } of distances) {
// If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
if (nearest.size < ef || distance < farthestInNearest[1]) {
candidates.set(id, distance)
nearest.set(id, distance)
// Apply filter if provided
const passes = filter ? await filter(id) : true
// Always add to candidates for graph traversal
candidates.set(id, distance)
// Only add to nearest if it passes the filter
if (passes) {
// If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
if (nearest.size < ef || distance < farthestInNearest[1]) {
nearest.set(id, distance)
// If we have more than ef neighbors, remove the farthest one
if (nearest.size > ef) {
const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
nearest.clear()
for (let i = 0; i < ef; i++) {
nearest.set(sortedNearest[i][0], sortedNearest[i][1])
// If we have more than ef neighbors, remove the farthest one
if (nearest.size > ef) {
const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
nearest.clear()
for (let i = 0; i < ef; i++) {
nearest.set(sortedNearest[i][0], sortedNearest[i][1])
}
}
}
}
@ -691,18 +704,26 @@ export class HNSWIndex {
queryVector,
neighbor.vector
)
// Apply filter if provided
const passes = filter ? await filter(neighborId) : true
// Always add to candidates for graph traversal
candidates.set(neighborId, distToNeighbor)
// Only add to nearest if it passes the filter
if (passes) {
// If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
if (nearest.size < ef || distToNeighbor < farthestInNearest[1]) {
nearest.set(neighborId, distToNeighbor)
// If we haven't found ef nearest neighbors yet, or this neighbor is closer than the farthest one we've found
if (nearest.size < ef || distToNeighbor < farthestInNearest[1]) {
candidates.set(neighborId, distToNeighbor)
nearest.set(neighborId, distToNeighbor)
// If we have more than ef neighbors, remove the farthest one
if (nearest.size > ef) {
const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
nearest.clear()
for (let i = 0; i < ef; i++) {
nearest.set(sortedNearest[i][0], sortedNearest[i][1])
// If we have more than ef neighbors, remove the farthest one
if (nearest.size > ef) {
const sortedNearest = [...nearest].sort((a, b) => a[1] - b[1])
nearest.clear()
for (let i = 0; i < ef; i++) {
nearest.set(sortedNearest[i][0], sortedNearest[i][1])
}
}
}
}

7
src/hnsw/optimizedHNSWIndex.ts
View file

@ -131,7 +131,8 @@ export class OptimizedHNSWIndex extends HNSWIndex {
*/
public async search(
queryVector: Vector,
k: number = 10
k: number = 10,
filter?: (id: string) => Promise<boolean>
): Promise<Array<[string, number]>> {
const startTime = Date.now()
@ -160,10 +161,10 @@ export class OptimizedHNSWIndex extends HNSWIndex {
try {
// This is a simplified approach - in practice, we'd need to modify
// the parent class to accept runtime parameter changes
results = await super.search(queryVector, k)
results = await super.search(queryVector, k, filter)
} catch (error) {
console.error('Optimized search failed, falling back to default:', error)
results = await super.search(queryVector, k)
results = await super.search(queryVector, k, filter)
}
// Record performance metrics

7
src/storage/adapters/optimizedS3Search.ts
View file

@ -5,6 +5,7 @@
import { HNSWNoun, GraphVerb } from '../../coreTypes.js'
import { createModuleLogger } from '../../utils/logger.js'
import { getDirectoryPath } from '../baseStorage.js'
const logger = createModuleLogger('OptimizedS3Search')
@ -70,7 +71,7 @@ export class OptimizedS3Search {
try {
// List noun objects with pagination
const listResult = await this.storage.listObjectKeys('nouns/', limit * 2, cursor)
const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('noun', 'vector')}/`, limit * 2, cursor)
if (!listResult.keys.length) {
return {
@ -141,7 +142,7 @@ export class OptimizedS3Search {
try {
// List verb objects with pagination
const listResult = await this.storage.listObjectKeys('verbs/', limit * 2, cursor)
const listResult = await this.storage.listObjectKeys(`${getDirectoryPath('verb', 'vector')}/`, limit * 2, cursor)
if (!listResult.keys.length) {
return {
@ -163,7 +164,7 @@ export class OptimizedS3Search {
if (!verbData) return null
// Get metadata
const verbId = key.replace('verbs/', '').replace('.json', '')
const verbId = key.replace(`${getDirectoryPath('verb', 'vector')}/`, '').replace('.json', '')
const metadata = await this.storage.getMetadata(verbId, 'verb')
// Combine into GraphVerb

36
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -12,7 +12,8 @@ import {
METADATA_DIR,
INDEX_DIR,
SYSTEM_DIR,
STATISTICS_KEY
STATISTICS_KEY,
getDirectoryPath
} from '../baseStorage.js'
import { StorageCompatibilityLayer, StoragePaths } from '../backwardCompatibility.js'
import {
@ -80,7 +81,8 @@ export class S3CompatibleStorage extends BaseStorage {
// Prefixes for different types of data
private nounPrefix: string
private verbPrefix: string
private metadataPrefix: string
private metadataPrefix: string // Noun metadata
private verbMetadataPrefix: string // Verb metadata
private indexPrefix: string // Legacy - for backward compatibility
private systemPrefix: string // New location for system data
private useDualWrite: boolean = true // Write to both locations during migration
@ -142,10 +144,11 @@ export class S3CompatibleStorage extends BaseStorage {
options.operationConfig
)
// Set up prefixes for different types of data
this.nounPrefix = `${NOUNS_DIR}/`
this.verbPrefix = `${VERBS_DIR}/`
this.metadataPrefix = `${METADATA_DIR}/`
// Set up prefixes for different types of data using new entity-based structure
this.nounPrefix = `${getDirectoryPath('noun', 'vector')}/`
this.verbPrefix = `${getDirectoryPath('verb', 'vector')}/`
this.metadataPrefix = `${getDirectoryPath('noun', 'metadata')}/` // Noun metadata
this.verbMetadataPrefix = `${getDirectoryPath('verb', 'metadata')}/` // Verb metadata
this.indexPrefix = `${INDEX_DIR}/` // Legacy
this.systemPrefix = `${SYSTEM_DIR}/` // New
@ -1283,7 +1286,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Import the PutObjectCommand only when needed
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
const key = `verb-metadata/${id}.json`
const key = `${this.verbMetadataPrefix}${id}.json`
const body = JSON.stringify(metadata, null, 2)
this.logger.trace(`Saving verb metadata for ${id} to key: ${key}`)
@ -1315,7 +1318,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
const key = `verb-metadata/${id}.json`
const key = `${this.verbMetadataPrefix}${id}.json`
this.logger.trace(`Getting verb metadata for ${id} from key: ${key}`)
// Try to get the verb metadata
@ -1373,7 +1376,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Import the PutObjectCommand only when needed
const { PutObjectCommand } = await import('@aws-sdk/client-s3')
const key = `noun-metadata/${id}.json`
const key = `${this.metadataPrefix}${id}.json`
const body = JSON.stringify(metadata, null, 2)
this.logger.trace(`Saving noun metadata for ${id} to key: ${key}`)
@ -1405,7 +1408,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Import the GetObjectCommand only when needed
const { GetObjectCommand } = await import('@aws-sdk/client-s3')
const key = `noun-metadata/${id}.json`
const key = `${this.metadataPrefix}${id}.json`
this.logger.trace(`Getting noun metadata for ${id} from key: ${key}`)
// Try to get the noun metadata
@ -1569,9 +1572,12 @@ export class S3CompatibleStorage extends BaseStorage {
// Delete all objects in the verbs directory
await deleteObjectsWithPrefix(this.verbPrefix)
// Delete all objects in the metadata directory
// Delete all objects in the noun metadata directory
await deleteObjectsWithPrefix(this.metadataPrefix)
// Delete all objects in the verb metadata directory
await deleteObjectsWithPrefix(this.verbMetadataPrefix)
// Delete all objects in the index directory
await deleteObjectsWithPrefix(this.indexPrefix)
@ -1659,17 +1665,19 @@ export class S3CompatibleStorage extends BaseStorage {
// Calculate size and count for each directory
const nounsResult = await calculateSizeAndCount(this.nounPrefix)
const verbsResult = await calculateSizeAndCount(this.verbPrefix)
const metadataResult = await calculateSizeAndCount(this.metadataPrefix)
const nounMetadataResult = await calculateSizeAndCount(this.metadataPrefix)
const verbMetadataResult = await calculateSizeAndCount(this.verbMetadataPrefix)
const indexResult = await calculateSizeAndCount(this.indexPrefix)
totalSize =
nounsResult.size +
verbsResult.size +
metadataResult.size +
nounMetadataResult.size +
verbMetadataResult.size +
indexResult.size
nodeCount = nounsResult.count
edgeCount = verbsResult.count
metadataCount = metadataResult.count
metadataCount = nounMetadataResult.count + verbMetadataResult.count
// Ensure we have a minimum size if we have objects
if (

48
src/storage/baseStorage.ts
View file

@ -7,17 +7,51 @@ import { GraphVerb, HNSWNoun, HNSWVerb, StatisticsData } from '../coreTypes.js'
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
// Common directory/prefix names
export const NOUNS_DIR = 'nouns'
export const VERBS_DIR = 'verbs'
export const METADATA_DIR = 'metadata'
export const NOUN_METADATA_DIR = 'noun-metadata'
export const VERB_METADATA_DIR = 'verb-metadata'
// Option A: Entity-Based Directory Structure
export const ENTITIES_DIR = 'entities'
export const NOUNS_VECTOR_DIR = 'entities/nouns/vectors'
export const NOUNS_METADATA_DIR = 'entities/nouns/metadata'
export const VERBS_VECTOR_DIR = 'entities/verbs/vectors'
export const VERBS_METADATA_DIR = 'entities/verbs/metadata'
export const INDEXES_DIR = 'indexes'
export const METADATA_INDEX_DIR = 'indexes/metadata'
// Legacy paths - kept for backward compatibility during migration
export const NOUNS_DIR = 'nouns' // Legacy: now maps to entities/nouns/vectors
export const VERBS_DIR = 'verbs' // Legacy: now maps to entities/verbs/vectors
export const METADATA_DIR = 'metadata' // Legacy: now maps to entities/nouns/metadata
export const NOUN_METADATA_DIR = 'noun-metadata' // Legacy: now maps to entities/nouns/metadata
export const VERB_METADATA_DIR = 'verb-metadata' // Legacy: now maps to entities/verbs/metadata
export const INDEX_DIR = 'index' // Legacy - kept for backward compatibility
export const SYSTEM_DIR = '_system' // New location for system data
export const SYSTEM_DIR = '_system' // System config & metadata indexes
export const STATISTICS_KEY = 'statistics'
// Migration version to track compatibility
export const STORAGE_SCHEMA_VERSION = 2 // Increment when making breaking changes
export const STORAGE_SCHEMA_VERSION = 3 // v3: Entity-Based Directory Structure (Option A)
// Configuration flag to enable new directory structure
export const USE_ENTITY_BASED_STRUCTURE = true // Set to true to use Option A structure
/**
* Get the appropriate directory path based on configuration
*/
export function getDirectoryPath(entityType: 'noun' | 'verb', dataType: 'vector' | 'metadata'): string {
if (USE_ENTITY_BASED_STRUCTURE) {
// Option A: Entity-Based Structure
if (entityType === 'noun') {
return dataType === 'vector' ? NOUNS_VECTOR_DIR : NOUNS_METADATA_DIR
} else {
return dataType === 'vector' ? VERBS_VECTOR_DIR : VERBS_METADATA_DIR
}
} else {
// Legacy structure
if (entityType === 'noun') {
return dataType === 'vector' ? NOUNS_DIR : METADATA_DIR
} else {
return dataType === 'vector' ? VERBS_DIR : VERB_METADATA_DIR
}
}
}
/**
* Base storage adapter that implements common functionality

321
src/utils/metadataFilter.ts Normal file
View file

@ -0,0 +1,321 @@
/**
* Smart metadata filtering for vector search
* Filters DURING search to ensure relevant results
* Simple API that just works without configuration
*/
import { SearchResult, HNSWNoun } from '../coreTypes.js'
/**
* MongoDB-style query operators
*/
export interface QueryOperators {
$eq?: any
$ne?: any
$gt?: any
$gte?: any
$lt?: any
$lte?: any
$in?: any[]
$nin?: any[]
$exists?: boolean
$regex?: string | RegExp
$includes?: any
$all?: any[]
$size?: number
$and?: MetadataFilter[]
$or?: MetadataFilter[]
$not?: MetadataFilter
}
/**
* Metadata filter definition
*/
export interface MetadataFilter {
[key: string]: any | QueryOperators
}
/**
* Options for metadata filtering
*/
export interface MetadataFilterOptions {
metadata?: MetadataFilter
scoring?: {
vectorWeight?: number
metadataWeight?: number
metadataBoosts?: Record<string, number | ((value: any, query: any) => number)>
}
}
/**
* Check if a value matches a query with operators
*/
function matchesQuery(value: any, query: any): boolean {
// Direct equality check
if (typeof query !== 'object' || query === null || Array.isArray(query)) {
return value === query
}
// Check for MongoDB-style operators
for (const [op, operand] of Object.entries(query)) {
switch (op) {
case '$eq':
if (value !== operand) return false
break
case '$ne':
if (value === operand) return false
break
case '$gt':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value > operand)) return false
break
case '$gte':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value >= operand)) return false
break
case '$lt':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value < operand)) return false
break
case '$lte':
if (typeof value !== 'number' || typeof operand !== 'number' || !(value <= operand)) return false
break
case '$in':
if (!Array.isArray(operand) || !operand.includes(value)) return false
break
case '$nin':
if (!Array.isArray(operand) || operand.includes(value)) return false
break
case '$exists':
if ((value !== undefined) !== operand) return false
break
case '$regex':
const regex = typeof operand === 'string' ? new RegExp(operand) : operand as RegExp
if (!(regex instanceof RegExp) || !regex.test(String(value))) return false
break
case '$includes':
if (!Array.isArray(value) || !value.includes(operand)) return false
break
case '$all':
if (!Array.isArray(value) || !Array.isArray(operand)) return false
for (const item of operand) {
if (!value.includes(item)) return false
}
break
case '$size':
if (!Array.isArray(value) || value.length !== operand) return false
break
default:
// Unknown operator, treat as field name
if (!matchesFieldQuery(value, op, operand)) return false
}
}
return true
}
/**
* Check if a field matches a query
*/
function matchesFieldQuery(obj: any, field: string, query: any): boolean {
const value = getNestedValue(obj, field)
return matchesQuery(value, query)
}
/**
* Get nested value from object using dot notation
*/
function getNestedValue(obj: any, path: string): any {
const parts = path.split('.')
let current = obj
for (const part of parts) {
if (current === null || current === undefined) {
return undefined
}
current = current[part]
}
return current
}
/**
* Check if metadata matches the filter
*/
export function matchesMetadataFilter(metadata: any, filter: MetadataFilter): boolean {
if (!filter || Object.keys(filter).length === 0) {
return true
}
for (const [key, query] of Object.entries(filter)) {
// Handle logical operators
if (key === '$and') {
if (!Array.isArray(query)) return false
for (const subFilter of query) {
if (!matchesMetadataFilter(metadata, subFilter)) return false
}
continue
}
if (key === '$or') {
if (!Array.isArray(query)) return false
let matched = false
for (const subFilter of query) {
if (matchesMetadataFilter(metadata, subFilter)) {
matched = true
break
}
}
if (!matched) return false
continue
}
if (key === '$not') {
if (matchesMetadataFilter(metadata, query)) return false
continue
}
// Handle field queries
const value = getNestedValue(metadata, key)
if (!matchesQuery(value, query)) {
return false
}
}
return true
}
/**
* Calculate metadata boost score
*/
export function calculateMetadataScore(
metadata: any,
filter: MetadataFilter,
scoring?: MetadataFilterOptions['scoring']
): number {
if (!scoring || !scoring.metadataBoosts) {
return 0
}
let score = 0
for (const [field, boost] of Object.entries(scoring.metadataBoosts)) {
const value = getNestedValue(metadata, field)
if (typeof boost === 'function') {
score += boost(value, filter)
} else if (value !== undefined) {
// Check if the field matches the filter
const fieldFilter = filter[field]
if (fieldFilter && matchesQuery(value, fieldFilter)) {
score += boost
}
}
}
return score
}
/**
* Apply compound scoring to search results
*/
export function applyCompoundScoring<T>(
results: SearchResult<T>[],
filter: MetadataFilter,
scoring?: MetadataFilterOptions['scoring']
): SearchResult<T>[] {
if (!scoring || (!scoring.vectorWeight && !scoring.metadataWeight)) {
return results
}
const vectorWeight = scoring.vectorWeight ?? 1.0
const metadataWeight = scoring.metadataWeight ?? 0.0
return results.map(result => {
const metadataScore = calculateMetadataScore(result.metadata, filter, scoring)
const combinedScore = (result.score * vectorWeight) + (metadataScore * metadataWeight)
return {
...result,
score: combinedScore
}
}).sort((a, b) => b.score - a.score) // Re-sort by combined score
}
/**
* Filter search results by metadata
*/
export function filterSearchResultsByMetadata<T>(
results: SearchResult<T>[],
filter: MetadataFilter
): SearchResult<T>[] {
if (!filter || Object.keys(filter).length === 0) {
return results
}
return results.filter(result =>
matchesMetadataFilter(result.metadata, filter)
)
}
/**
* Filter nouns by metadata before search
*/
export function filterNounsByMetadata(
nouns: HNSWNoun[],
filter: MetadataFilter
): HNSWNoun[] {
if (!filter || Object.keys(filter).length === 0) {
return nouns
}
return nouns.filter(noun =>
matchesMetadataFilter(noun.metadata, filter)
)
}
/**
* Aggregate search results for faceted search
*/
export interface FacetConfig {
field: string
limit?: number
}
export interface FacetResult {
[value: string]: number
}
export interface AggregationResult<T> {
results: SearchResult<T>[]
facets: Record<string, FacetResult>
}
export function aggregateSearchResults<T>(
results: SearchResult<T>[],
facets: Record<string, FacetConfig>
): AggregationResult<T> {
const facetResults: Record<string, FacetResult> = {}
for (const [facetName, config] of Object.entries(facets)) {
const counts: Record<string, number> = {}
for (const result of results) {
const value = getNestedValue(result.metadata, config.field)
if (value !== undefined) {
const key = String(value)
counts[key] = (counts[key] || 0) + 1
}
}
// Sort by count and apply limit
const sorted = Object.entries(counts)
.sort((a, b) => b[1] - a[1])
.slice(0, config.limit || 10)
facetResults[facetName] = Object.fromEntries(sorted)
}
return {
results,
facets: facetResults
}
}

451
src/utils/metadataIndex.ts Normal file
View file

@ -0,0 +1,451 @@
/**
* Metadata Index System
* Maintains inverted indexes for fast metadata filtering
* Automatically updates indexes when data changes
*/
import { StorageAdapter } from '../coreTypes.js'
export interface MetadataIndexEntry {
field: string
value: string | number | boolean
ids: Set<string>
lastUpdated: number
}
export interface MetadataIndexStats {
totalEntries: number
totalIds: number
fieldsIndexed: string[]
lastRebuild: number
indexSize: number // in bytes
}
export interface MetadataIndexConfig {
maxIndexSize?: number // Max number of entries per field value (default: 10000)
rebuildThreshold?: number // Rebuild if index is this % stale (default: 0.1)
autoOptimize?: boolean // Auto-cleanup unused entries (default: true)
indexedFields?: string[] // Only index these fields (default: all)
excludeFields?: string[] // Never index these fields
}
/**
* Manages metadata indexes for fast filtering
* Maintains inverted indexes: field+value -> list of IDs
*/
export class MetadataIndexManager {
private storage: StorageAdapter
private config: Required<MetadataIndexConfig>
private indexCache = new Map<string, MetadataIndexEntry>()
private dirtyEntries = new Set<string>()
private isRebuilding = false
constructor(storage: StorageAdapter, config: MetadataIndexConfig = {}) {
this.storage = storage
this.config = {
maxIndexSize: config.maxIndexSize ?? 10000,
rebuildThreshold: config.rebuildThreshold ?? 0.1,
autoOptimize: config.autoOptimize ?? true,
indexedFields: config.indexedFields ?? [],
excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt']
}
}
/**
* Get index key for field and value
*/
private getIndexKey(field: string, value: any): string {
const normalizedValue = this.normalizeValue(value)
return `${field}:${normalizedValue}`
}
/**
* Normalize value for consistent indexing
*/
private normalizeValue(value: any): string {
if (value === null || value === undefined) return '__NULL__'
if (typeof value === 'boolean') return value ? '__TRUE__' : '__FALSE__'
if (typeof value === 'number') return value.toString()
if (Array.isArray(value)) return value.map(v => this.normalizeValue(v)).join(',')
return String(value).toLowerCase().trim()
}
/**
* Check if field should be indexed
*/
private shouldIndexField(field: string): boolean {
if (this.config.excludeFields.includes(field)) return false
if (this.config.indexedFields.length > 0) {
return this.config.indexedFields.includes(field)
}
return true
}
/**
* Extract indexable field-value pairs from metadata
*/
private extractIndexableFields(metadata: any): Array<{ field: string, value: any }> {
const fields: Array<{ field: string, value: any }> = []
const extract = (obj: any, prefix = ''): void => {
for (const [key, value] of Object.entries(obj)) {
const fullKey = prefix ? `${prefix}.${key}` : key
if (!this.shouldIndexField(fullKey)) continue
if (value && typeof value === 'object' && !Array.isArray(value)) {
// Recurse into nested objects
extract(value, fullKey)
} else {
// Index this field
fields.push({ field: fullKey, value })
// If it's an array, also index each element
if (Array.isArray(value)) {
for (const item of value) {
fields.push({ field: fullKey, value: item })
}
}
}
}
}
if (metadata && typeof metadata === 'object') {
extract(metadata)
}
return fields
}
/**
* Add item to metadata indexes
*/
async addToIndex(id: string, metadata: any): Promise<void> {
const fields = this.extractIndexableFields(metadata)
for (const { field, value } of fields) {
const key = this.getIndexKey(field, value)
// Get or create index entry
let entry = this.indexCache.get(key)
if (!entry) {
const loadedEntry = await this.loadIndexEntry(key)
entry = loadedEntry ?? {
field,
value: this.normalizeValue(value),
ids: new Set<string>(),
lastUpdated: Date.now()
}
this.indexCache.set(key, entry)
}
// Add ID to entry
entry.ids.add(id)
entry.lastUpdated = Date.now()
this.dirtyEntries.add(key)
}
}
/**
* Remove item from metadata indexes
*/
async removeFromIndex(id: string, metadata?: any): Promise<void> {
if (metadata) {
// Remove from specific field indexes
const fields = this.extractIndexableFields(metadata)
for (const { field, value } of fields) {
const key = this.getIndexKey(field, value)
let entry = this.indexCache.get(key)
if (!entry) {
const loadedEntry = await this.loadIndexEntry(key)
entry = loadedEntry ?? undefined
}
if (entry) {
entry.ids.delete(id)
entry.lastUpdated = Date.now()
this.dirtyEntries.add(key)
// If no IDs left, mark for cleanup
if (entry.ids.size === 0) {
this.indexCache.delete(key)
await this.deleteIndexEntry(key)
}
}
}
} else {
// Remove from all indexes (slower, requires scanning)
for (const [key, entry] of this.indexCache.entries()) {
if (entry.ids.has(id)) {
entry.ids.delete(id)
entry.lastUpdated = Date.now()
this.dirtyEntries.add(key)
if (entry.ids.size === 0) {
this.indexCache.delete(key)
await this.deleteIndexEntry(key)
}
}
}
}
}
/**
* Get IDs for a specific field-value combination
*/
async getIds(field: string, value: any): Promise<string[]> {
const key = this.getIndexKey(field, value)
// Try cache first
let entry = this.indexCache.get(key)
// Load from storage if not cached
if (!entry) {
const loadedEntry = await this.loadIndexEntry(key)
if (loadedEntry) {
entry = loadedEntry
this.indexCache.set(key, entry)
}
}
return entry ? Array.from(entry.ids) : []
}
/**
* Convert MongoDB-style filter to simple field-value criteria for indexing
*/
private convertFilterToCriteria(filter: any): Array<{ field: string, values: any[] }> {
const criteria: Array<{ field: string, values: any[] }> = []
if (!filter || typeof filter !== 'object') {
return criteria
}
for (const [key, value] of Object.entries(filter)) {
// Skip logical operators for now - handle them separately
if (key.startsWith('$')) continue
if (value && typeof value === 'object' && !Array.isArray(value)) {
// Handle MongoDB operators
for (const [op, operand] of Object.entries(value)) {
switch (op) {
case '$in':
if (Array.isArray(operand)) {
criteria.push({ field: key, values: operand })
}
break
case '$eq':
criteria.push({ field: key, values: [operand] })
break
// For other operators, we can't use index efficiently, skip for now
default:
break
}
}
} else {
// Direct value or array
const values = Array.isArray(value) ? value : [value]
criteria.push({ field: key, values })
}
}
return criteria
}
/**
* Get IDs matching MongoDB-style metadata filter using indexes where possible
*/
async getIdsForFilter(filter: any): Promise<string[]> {
if (!filter || Object.keys(filter).length === 0) {
return []
}
// Handle logical operators
if (filter.$and && Array.isArray(filter.$and)) {
// For $and, we need intersection of all sub-filters
const allIds: string[][] = []
for (const subFilter of filter.$and) {
const subIds = await this.getIdsForFilter(subFilter)
allIds.push(subIds)
}
if (allIds.length === 0) return []
if (allIds.length === 1) return allIds[0]
// Intersection of all sets
return allIds.reduce((intersection, currentSet) =>
intersection.filter(id => currentSet.includes(id))
)
}
if (filter.$or && Array.isArray(filter.$or)) {
// For $or, we need union of all sub-filters
const unionIds = new Set<string>()
for (const subFilter of filter.$or) {
const subIds = await this.getIdsForFilter(subFilter)
subIds.forEach(id => unionIds.add(id))
}
return Array.from(unionIds)
}
// Handle regular field filters
const criteria = this.convertFilterToCriteria(filter)
const idSets: string[][] = []
for (const { field, values } of criteria) {
const unionIds = new Set<string>()
for (const value of values) {
const ids = await this.getIds(field, value)
ids.forEach(id => unionIds.add(id))
}
idSets.push(Array.from(unionIds))
}
if (idSets.length === 0) return []
if (idSets.length === 1) return idSets[0]
// Intersection of all field criteria (implicit $and)
return idSets.reduce((intersection, currentSet) =>
intersection.filter(id => currentSet.includes(id))
)
}
/**
* Get IDs matching multiple criteria (intersection) - LEGACY METHOD
* @deprecated Use getIdsForFilter instead
*/
async getIdsForCriteria(criteria: Record<string, any>): Promise<string[]> {
return this.getIdsForFilter(criteria)
}
/**
* Flush dirty entries to storage
*/
async flush(): Promise<void> {
const promises: Promise<void>[] = []
for (const key of this.dirtyEntries) {
const entry = this.indexCache.get(key)
if (entry) {
promises.push(this.saveIndexEntry(key, entry))
}
}
await Promise.all(promises)
this.dirtyEntries.clear()
}
/**
* Get index statistics
*/
async getStats(): Promise<MetadataIndexStats> {
const fields = new Set<string>()
let totalEntries = 0
let totalIds = 0
for (const entry of this.indexCache.values()) {
fields.add(entry.field)
totalEntries++
totalIds += entry.ids.size
}
return {
totalEntries,
totalIds,
fieldsIndexed: Array.from(fields),
lastRebuild: 0, // TODO: track rebuild timestamp
indexSize: totalEntries * 100 // rough estimate
}
}
/**
* Rebuild entire index from scratch
*/
async rebuild(): Promise<void> {
if (this.isRebuilding) return
this.isRebuilding = true
try {
// Clear existing indexes
this.indexCache.clear()
this.dirtyEntries.clear()
// Get all nouns and rebuild their metadata indexes
const nouns = await this.storage.getAllNouns()
for (const noun of nouns) {
const metadata = await this.storage.getMetadata(noun.id)
if (metadata) {
await this.addToIndex(noun.id, metadata)
}
}
// Get all verbs and rebuild their metadata indexes
const verbs = await this.storage.getAllVerbs()
for (const verb of verbs) {
const metadata = await this.storage.getVerbMetadata(verb.id)
if (metadata) {
await this.addToIndex(verb.id, metadata)
}
}
// Flush to storage
await this.flush()
} finally {
this.isRebuilding = false
}
}
/**
* Load index entry from storage
*/
private async loadIndexEntry(key: string): Promise<MetadataIndexEntry | null> {
try {
// Load metadata indexes from the _system directory with a special prefix
const indexId = `__metadata_index__${key}`
const data = await this.storage.getMetadata(indexId)
if (data) {
return {
field: data.field,
value: data.value,
ids: new Set(data.ids || []),
lastUpdated: data.lastUpdated || Date.now()
}
}
} catch (error) {
// Index entry doesn't exist yet
}
return null
}
/**
* Save index entry to storage
*/
private async saveIndexEntry(key: string, entry: MetadataIndexEntry): Promise<void> {
const data = {
field: entry.field,
value: entry.value,
ids: Array.from(entry.ids),
lastUpdated: entry.lastUpdated
}
// Store metadata indexes in the _system directory with a special prefix
const indexId = `__metadata_index__${key}`
await this.storage.saveMetadata(indexId, data)
}
/**
* Delete index entry from storage
*/
private async deleteIndexEntry(key: string): Promise<void> {
try {
const indexId = `__metadata_index__${key}`
await this.storage.saveMetadata(indexId, null)
} catch (error) {
// Entry might not exist
}
}
}