feat: v0.49 - Filter discovery API, remove deprecated methods, improve performance
BREAKING CHANGES: - Removed deprecated getAllNouns() and getAllVerbs() methods - All internal usage migrated to pagination-based methods New Features: - Filter Discovery API: - getFilterValues(field): Get all available values for a field - getFilterFields(): Get all filterable fields - Enables dynamic filter UI generation with O(1) field discovery - Hybrid metadata indexing with field-level indexes - Adaptive auto-flush for optimal performance - LRU caching for metadata indexes Improvements: - Fixed ENAMETOOLONG errors from vector-based filenames - Safe filename generation using hash-based approach - Scalable chunked value storage for millions of entries - Performance optimization with adaptive flush thresholds - Added support for $includes operator in metadata filters Technical: - Replaced vector-based filenames with safe hash approach - Implemented MetadataIndexCache with existing SearchCache pattern - Field indexes enable O(1) filter discovery - Adaptive flush based on performance metrics (20-200 entries) - All tests passing with improved metadata filtering
This commit is contained in:
parent
ac5b3183e3
commit
2dc909909a
17 changed files with 942 additions and 486 deletions
|
|
@ -980,53 +980,66 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// If the noun count has changed, update the index
|
||||
if (currentCount !== this.lastKnownNounCount) {
|
||||
// Get all nouns from storage
|
||||
const nouns = await this.storage!.getAllNouns()
|
||||
|
||||
// Get all nouns currently in the index
|
||||
const indexNouns = this.index.getNouns()
|
||||
const indexNounIds = new Set(indexNouns.keys())
|
||||
|
||||
// Find nouns that are in storage but not in the index
|
||||
const newNouns = nouns.filter((noun) => !indexNounIds.has(noun.id))
|
||||
|
||||
// Add new nouns to the index
|
||||
for (const noun of newNouns) {
|
||||
// Check if the vector dimensions match the expected dimensions
|
||||
if (noun.vector.length !== this._dimensions) {
|
||||
console.warn(
|
||||
`Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
// Add to index
|
||||
await this.index.addItem({
|
||||
id: noun.id,
|
||||
vector: noun.vector
|
||||
// Use pagination to load nouns from storage
|
||||
let offset = 0
|
||||
const limit = 100
|
||||
let hasMore = true
|
||||
let totalNewNouns = 0
|
||||
|
||||
while (hasMore) {
|
||||
const result = await this.storage!.getNouns({
|
||||
pagination: { offset, limit }
|
||||
})
|
||||
|
||||
// Find nouns that are in storage but not in the index
|
||||
const newNouns = result.items.filter((noun) => !indexNounIds.has(noun.id))
|
||||
totalNewNouns += newNouns.length
|
||||
|
||||
// Add new nouns to the index
|
||||
for (const noun of newNouns) {
|
||||
// Check if the vector dimensions match the expected dimensions
|
||||
if (noun.vector.length !== this._dimensions) {
|
||||
console.warn(
|
||||
`Skipping noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(
|
||||
`Added new noun ${noun.id} to index during real-time update`
|
||||
)
|
||||
// Add to index
|
||||
await this.index.addItem({
|
||||
id: noun.id,
|
||||
vector: noun.vector
|
||||
})
|
||||
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log(
|
||||
`Added new noun ${noun.id} to index during real-time update`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
hasMore = result.hasMore
|
||||
offset += limit
|
||||
}
|
||||
|
||||
// Update the last known noun count
|
||||
this.lastKnownNounCount = currentCount
|
||||
|
||||
// Invalidate search cache if new nouns were detected
|
||||
if (newNouns.length > 0) {
|
||||
if (totalNewNouns > 0) {
|
||||
this.searchCache.invalidateOnDataChange('add')
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log('Search cache invalidated due to external data changes')
|
||||
}
|
||||
}
|
||||
|
||||
if (this.loggingConfig?.verbose && newNouns.length > 0) {
|
||||
if (this.loggingConfig?.verbose && totalNewNouns > 0) {
|
||||
console.log(
|
||||
`Real-time update: Added ${newNouns.length} new nouns to index using full scan`
|
||||
`Real-time update: Added ${totalNewNouns} new nouns to index using full scan`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -1212,27 +1225,38 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Just initialize an empty index
|
||||
this.index.clear()
|
||||
} else {
|
||||
// Load all nouns from storage
|
||||
const nouns: HNSWNoun[] = await this.storage!.getAllNouns()
|
||||
|
||||
// Clear the index and add all nouns
|
||||
// Clear the index and load nouns using pagination
|
||||
this.index.clear()
|
||||
for (const noun of nouns) {
|
||||
// Check if the vector dimensions match the expected dimensions
|
||||
if (noun.vector.length !== this._dimensions) {
|
||||
console.warn(
|
||||
`Deleting noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`
|
||||
)
|
||||
// Delete the mismatched noun from storage to prevent future issues
|
||||
await this.storage!.deleteNoun(noun.id)
|
||||
continue
|
||||
}
|
||||
|
||||
// Add to index
|
||||
await this.index.addItem({
|
||||
id: noun.id,
|
||||
vector: noun.vector
|
||||
|
||||
let offset = 0
|
||||
const limit = 100
|
||||
let hasMore = true
|
||||
|
||||
while (hasMore) {
|
||||
const result = await this.storage!.getNouns({
|
||||
pagination: { offset, limit }
|
||||
})
|
||||
|
||||
for (const noun of result.items) {
|
||||
// Check if the vector dimensions match the expected dimensions
|
||||
if (noun.vector.length !== this._dimensions) {
|
||||
console.warn(
|
||||
`Deleting noun ${noun.id} due to dimension mismatch: expected ${this._dimensions}, got ${noun.vector.length}`
|
||||
)
|
||||
// Delete the mismatched noun from storage to prevent future issues
|
||||
await this.storage!.deleteNoun(noun.id)
|
||||
continue
|
||||
}
|
||||
|
||||
// Add to index
|
||||
await this.index.addItem({
|
||||
id: noun.id,
|
||||
vector: noun.vector
|
||||
})
|
||||
}
|
||||
|
||||
hasMore = result.hasMore
|
||||
offset += limit
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1267,15 +1291,31 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
)
|
||||
|
||||
// Check if we need to rebuild the index (for existing data)
|
||||
// Skip rebuild for memory storage (starts empty) or when in read-only mode
|
||||
// Also skip if index already has entries
|
||||
const isMemoryStorage = this.storage?.constructor?.name === 'MemoryStorage'
|
||||
const stats = await this.metadataIndex.getStats()
|
||||
if (stats.totalEntries === 0 && !this.readOnly) {
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log('Rebuilding metadata index for existing data...')
|
||||
}
|
||||
await this.metadataIndex.rebuild()
|
||||
if (this.loggingConfig?.verbose) {
|
||||
const newStats = await this.metadataIndex.getStats()
|
||||
console.log(`Metadata index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`)
|
||||
|
||||
if (!isMemoryStorage && !this.readOnly && stats.totalEntries === 0) {
|
||||
// Check if we have existing data that needs indexing
|
||||
// Use a simple check to avoid expensive operations
|
||||
try {
|
||||
const testResult = await this.storage!.getNouns({ pagination: { offset: 0, limit: 1 }})
|
||||
if (testResult.items.length > 0) {
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log('Rebuilding metadata index for existing data...')
|
||||
}
|
||||
await this.metadataIndex.rebuild()
|
||||
if (this.loggingConfig?.verbose) {
|
||||
const newStats = await this.metadataIndex.getStats()
|
||||
console.log(`Metadata index rebuilt: ${newStats.totalEntries} entries, ${newStats.fieldsIndexed.length} fields`)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// If getNouns fails, skip rebuild
|
||||
if (this.loggingConfig?.verbose) {
|
||||
console.log('Skipping metadata index rebuild:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2124,10 +2164,11 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
|
||||
// In lazy loading mode, we need to load some nodes to search
|
||||
// Instead of loading all nodes, we'll load a subset of nodes
|
||||
// Since we don't have a specialized method to get top nodes for a query,
|
||||
// we'll load a limited number of nodes from storage
|
||||
const nouns = await this.storage!.getAllNouns()
|
||||
const limitedNouns = nouns.slice(0, Math.min(nouns.length, k * 10)) // Get 10x more nodes than needed
|
||||
// Load a limited number of nodes from storage using pagination
|
||||
const result = await this.storage!.getNouns({
|
||||
pagination: { offset: 0, limit: k * 10 } // Get 10x more nodes than needed
|
||||
})
|
||||
const limitedNouns = result.items
|
||||
|
||||
// Add these nodes to the index
|
||||
for (const node of limitedNouns) {
|
||||
|
|
@ -2163,6 +2204,9 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
// Use metadata index for pre-filtering if available
|
||||
if (hasMetadataFilter && this.metadataIndex) {
|
||||
try {
|
||||
// Ensure metadata index is up to date
|
||||
await this.metadataIndex.flush()
|
||||
|
||||
// Get candidate IDs from metadata index
|
||||
const candidateIds = await this.metadataIndex.getIdsForFilter(options.metadata)
|
||||
if (candidateIds.length > 0) {
|
||||
|
|
@ -4717,7 +4761,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
const searchResults = await this.index.search(queryVector, k * 2)
|
||||
|
||||
// Get all verbs for filtering
|
||||
const allVerbs = await this.storage!.getAllVerbs()
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
|
||||
// Create a map of verb IDs for faster lookup
|
||||
const verbMap = new Map<string, GraphVerb>()
|
||||
|
|
@ -4925,6 +4969,39 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get available filter values for a field
|
||||
* Useful for building dynamic filter UIs
|
||||
*
|
||||
* @param field The field name to get values for
|
||||
* @returns Array of available values for that field
|
||||
*/
|
||||
public async getFilterValues(field: string): Promise<string[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
if (!this.metadataIndex) {
|
||||
return []
|
||||
}
|
||||
|
||||
return this.metadataIndex.getFilterValues(field)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all available filter fields
|
||||
* Useful for discovering what metadata fields are indexed
|
||||
*
|
||||
* @returns Array of indexed field names
|
||||
*/
|
||||
public async getFilterFields(): Promise<string[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
if (!this.metadataIndex) {
|
||||
return []
|
||||
}
|
||||
|
||||
return this.metadataIndex.getFilterFields()
|
||||
}
|
||||
|
||||
/**
|
||||
* Search within a specific set of items
|
||||
* This is useful when you've pre-filtered items and want to search only within them
|
||||
|
|
|
|||
|
|
@ -339,13 +339,6 @@ export interface StorageAdapter {
|
|||
|
||||
getNoun(id: string): Promise<HNSWNoun | null>
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* @deprecated Use getNouns() with pagination instead for better scalability
|
||||
* @returns Promise that resolves to an array of all nouns
|
||||
*/
|
||||
getAllNouns(): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
@ -383,13 +376,6 @@ export interface StorageAdapter {
|
|||
|
||||
getVerb(id: string): Promise<GraphVerb | null>
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
* @deprecated Use getVerbs() with pagination instead for better scalability
|
||||
* @returns Promise that resolves to an array of all verbs
|
||||
*/
|
||||
getAllVerbs(): Promise<GraphVerb[]>
|
||||
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
@ -562,4 +548,18 @@ export interface StorageAdapter {
|
|||
* @returns Promise that resolves to an array of changes
|
||||
*/
|
||||
getChangesSince?(timestamp: number, limit?: number): Promise<any[]>
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* @returns Promise that resolves to an array of all nouns
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
|
||||
*/
|
||||
getAllNouns(): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
* @returns Promise that resolves to an array of all HNSWVerbs
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
|
||||
*/
|
||||
getAllVerbs(): Promise<HNSWVerb[]>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,8 +17,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
|
||||
abstract getNoun(id: string): Promise<any | null>
|
||||
|
||||
abstract getAllNouns(): Promise<any[]>
|
||||
|
||||
abstract getNounsByNounType(nounType: string): Promise<any[]>
|
||||
|
||||
abstract deleteNoun(id: string): Promise<void>
|
||||
|
|
@ -27,8 +25,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
|
||||
abstract getVerb(id: string): Promise<any | null>
|
||||
|
||||
abstract getAllVerbs(): Promise<any[]>
|
||||
|
||||
abstract getVerbsBySource(sourceId: string): Promise<any[]>
|
||||
|
||||
abstract getVerbsByTarget(targetId: string): Promise<any[]>
|
||||
|
|
@ -54,6 +50,20 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
details?: Record<string, any>
|
||||
}>
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* @returns Promise that resolves to an array of all nouns
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
|
||||
*/
|
||||
abstract getAllNouns(): Promise<any[]>
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
* @returns Promise that resolves to an array of all HNSWVerbs
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
|
||||
*/
|
||||
abstract getAllVerbs(): Promise<any[]>
|
||||
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
|
|||
|
|
@ -752,12 +752,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
return this.getNode(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
||||
return this.getAllNodes()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
|
|
@ -789,12 +783,6 @@ export class FileSystemStorage extends BaseStorage {
|
|||
return this.getEdge(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs by source
|
||||
|
|
|
|||
|
|
@ -83,33 +83,6 @@ export class MemoryStorage extends BaseStorage {
|
|||
return nounCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
||||
const allNouns: HNSWNoun[] = []
|
||||
|
||||
// Iterate through all nouns in the nouns map
|
||||
for (const [nounId, noun] of this.nouns.entries()) {
|
||||
// Return a deep copy to avoid reference issues
|
||||
const nounCopy: HNSWNoun = {
|
||||
id: noun.id,
|
||||
vector: [...noun.vector],
|
||||
connections: new Map(),
|
||||
level: noun.level || 0
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of noun.connections.entries()) {
|
||||
nounCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allNouns.push(nounCopy)
|
||||
}
|
||||
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
@ -297,32 +270,6 @@ export class MemoryStorage extends BaseStorage {
|
|||
return verbCopy
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
|
||||
const allVerbs: HNSWVerb[] = []
|
||||
|
||||
// Iterate through all verbs in the verbs map
|
||||
for (const [verbId, verb] of this.verbs.entries()) {
|
||||
// Create a deep copy of the HNSWVerb
|
||||
const verbCopy: HNSWVerb = {
|
||||
id: verb.id,
|
||||
vector: [...verb.vector],
|
||||
connections: new Map()
|
||||
}
|
||||
|
||||
// Copy connections
|
||||
for (const [level, connections] of verb.connections.entries()) {
|
||||
verbCopy.connections.set(level, new Set(connections))
|
||||
}
|
||||
|
||||
allVerbs.push(verbCopy)
|
||||
}
|
||||
|
||||
return allVerbs
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verbs with pagination and filtering
|
||||
* @param options Pagination and filtering options
|
||||
|
|
|
|||
|
|
@ -255,46 +255,6 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
*/
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun_internal[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const allNouns: HNSWNoun_internal[] = []
|
||||
try {
|
||||
// Iterate through all files in the nouns directory
|
||||
for await (const [name, handle] of this.nounsDir!.entries()) {
|
||||
if (handle.kind === 'file') {
|
||||
try {
|
||||
// Read the noun data from the file
|
||||
const file = await safeGetFile(handle)
|
||||
const text = await file.text()
|
||||
const data = JSON.parse(text)
|
||||
|
||||
// Convert serialized connections back to Map<number, Set<string>>
|
||||
const connections = new Map<number, Set<string>>()
|
||||
for (const [level, nounIds] of Object.entries(data.connections)) {
|
||||
connections.set(Number(level), new Set(nounIds as string[]))
|
||||
}
|
||||
|
||||
allNouns.push({
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections,
|
||||
level: data.level || 0
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(`Error reading noun file ${name}:`, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error reading nouns directory:', error)
|
||||
}
|
||||
|
||||
return allNouns
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type (internal implementation)
|
||||
|
|
@ -469,12 +429,6 @@ export class OPFSStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage (internal implementation)
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
|
|
|
|||
|
|
@ -466,17 +466,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage (internal implementation)
|
||||
*/
|
||||
protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
|
||||
// Use paginated method to avoid deprecation warning
|
||||
const result = await this.getNodesWithPagination({
|
||||
limit: 1000,
|
||||
useCache: true
|
||||
})
|
||||
return result.nodes
|
||||
}
|
||||
|
||||
// Node cache to avoid redundant API calls
|
||||
private nodeCache = new Map<string, HNSWNode>()
|
||||
|
|
@ -854,15 +843,6 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all verbs from storage (internal implementation)
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getVerbsWithPagination() instead.
|
||||
*/
|
||||
protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
|
||||
this.logger.warn('getAllVerbs_internal() is deprecated and will be removed in a future version. Use getVerbsWithPagination() instead.')
|
||||
return this.getAllEdges()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all edges from storage
|
||||
|
|
|
|||
|
|
@ -92,17 +92,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
return this.getNoun_internal(id)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getNouns() with pagination instead.
|
||||
*/
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
console.warn('WARNING: getAllNouns() is deprecated and will be removed in a future version. Use getNouns() with pagination instead.')
|
||||
return this.getAllNouns_internal()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* @param nounType The noun type to filter by
|
||||
|
|
@ -215,24 +204,34 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
* @deprecated This method is deprecated and will be removed in a future version.
|
||||
* It can cause memory issues with large datasets. Use getVerbs() with pagination instead.
|
||||
* @returns Promise that resolves to an array of all HNSWVerbs
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
|
||||
*/
|
||||
public async getAllVerbs(): Promise<GraphVerb[]> {
|
||||
public async getAllVerbs(): Promise<HNSWVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
console.warn('WARNING: getAllVerbs() is deprecated and will be removed in a future version. Use getVerbs() with pagination instead.')
|
||||
|
||||
const hnswVerbs = await this.getAllVerbs_internal()
|
||||
const graphVerbs: GraphVerb[] = []
|
||||
console.warn('getAllVerbs() is deprecated and may cause memory issues with large datasets. Consider using getVerbs() with pagination instead.')
|
||||
|
||||
for (const hnswVerb of hnswVerbs) {
|
||||
const graphVerb = await this.convertHNSWVerbToGraphVerb(hnswVerb)
|
||||
if (graphVerb) {
|
||||
graphVerbs.push(graphVerb)
|
||||
// Get all verbs using the paginated method with a very large limit
|
||||
const result = await this.getVerbs({
|
||||
pagination: {
|
||||
limit: Number.MAX_SAFE_INTEGER
|
||||
}
|
||||
})
|
||||
|
||||
// Convert GraphVerbs back to HNSWVerbs since that's what this method should return
|
||||
const hnswVerbs: HNSWVerb[] = []
|
||||
for (const graphVerb of result.items) {
|
||||
// Create an HNSWVerb from the GraphVerb (reverse conversion)
|
||||
const hnswVerb: HNSWVerb = {
|
||||
id: graphVerb.id,
|
||||
vector: graphVerb.vector,
|
||||
connections: new Map() // HNSWVerbs need connections, but GraphVerbs don't have them
|
||||
}
|
||||
hnswVerbs.push(hnswVerb)
|
||||
}
|
||||
|
||||
return graphVerbs
|
||||
return hnswVerbs
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -241,11 +240,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async getVerbsBySource(sourceId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get all verbs and filter by source
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter(verb =>
|
||||
verb.sourceId === sourceId || verb.source === sourceId
|
||||
)
|
||||
// Use the paginated getVerbs method with source filter
|
||||
const result = await this.getVerbs({
|
||||
filter: { sourceId }
|
||||
})
|
||||
return result.items
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -254,11 +253,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async getVerbsByTarget(targetId: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get all verbs and filter by target
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter(verb =>
|
||||
verb.targetId === targetId || verb.target === targetId
|
||||
)
|
||||
// Use the paginated getVerbs method with target filter
|
||||
const result = await this.getVerbs({
|
||||
filter: { targetId }
|
||||
})
|
||||
return result.items
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -267,11 +266,30 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
public async getVerbsByType(type: string): Promise<GraphVerb[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Get all verbs and filter by type
|
||||
const allVerbs = await this.getAllVerbs()
|
||||
return allVerbs.filter(verb =>
|
||||
verb.type === type || verb.verb === type
|
||||
)
|
||||
// Use the paginated getVerbs method with type filter
|
||||
const result = await this.getVerbs({
|
||||
filter: { verbType: type }
|
||||
})
|
||||
return result.items
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* @returns Promise that resolves to an array of all nouns
|
||||
* @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
|
||||
*/
|
||||
public async getAllNouns(): Promise<HNSWNoun[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
console.warn('getAllNouns() is deprecated and may cause memory issues with large datasets. Consider using getNouns() with pagination instead.')
|
||||
|
||||
const result = await this.getNouns({
|
||||
pagination: {
|
||||
limit: Number.MAX_SAFE_INTEGER
|
||||
}
|
||||
})
|
||||
|
||||
return result.items
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -374,100 +392,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
// If the adapter doesn't have a paginated method, fall back to the old approach
|
||||
// but with a warning and a reasonable limit
|
||||
console.warn(
|
||||
'Storage adapter does not support pagination, falling back to loading all nouns. This may cause performance issues with large datasets.'
|
||||
// Storage adapter does not support pagination
|
||||
console.error(
|
||||
'Storage adapter does not support pagination. The deprecated getAllNouns_internal() method has been removed. Please implement getNounsWithPagination() in your storage adapter.'
|
||||
)
|
||||
|
||||
// Get nouns with a reasonable limit to avoid memory issues
|
||||
const maxNouns = Math.min(offset + limit + 100, 1000) // Reasonable limit
|
||||
let allNouns: HNSWNoun[] = []
|
||||
|
||||
try {
|
||||
// Try to get only the nouns we need
|
||||
allNouns = await this.getAllNouns_internal()
|
||||
|
||||
// If we have too many nouns, truncate the array to avoid memory issues
|
||||
if (allNouns.length > maxNouns) {
|
||||
console.warn(
|
||||
`Large number of nouns (${allNouns.length}), truncating to ${maxNouns} for filtering`
|
||||
)
|
||||
allNouns = allNouns.slice(0, maxNouns)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting all nouns:', error)
|
||||
// Return empty result on error
|
||||
return {
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filtering if needed
|
||||
let filteredNouns = allNouns
|
||||
|
||||
if (options?.filter) {
|
||||
// Filter by noun type
|
||||
if (options.filter.nounType) {
|
||||
const nounTypes = Array.isArray(options.filter.nounType)
|
||||
? options.filter.nounType
|
||||
: [options.filter.nounType]
|
||||
|
||||
filteredNouns = filteredNouns.filter((noun) => {
|
||||
// HNSWNoun doesn't have a type property directly, check metadata
|
||||
const nounType = noun.metadata?.type
|
||||
return typeof nounType === 'string' && nounTypes.includes(nounType)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
|
||||
filteredNouns = filteredNouns.filter((noun) => {
|
||||
// HNSWNoun doesn't have a service property directly, check metadata
|
||||
const service = noun.metadata?.service
|
||||
return typeof service === 'string' && services.includes(service)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata) {
|
||||
const metadataFilter = options.filter.metadata
|
||||
filteredNouns = filteredNouns.filter((noun) => {
|
||||
if (!noun.metadata) return false
|
||||
|
||||
// Check if all metadata keys match
|
||||
return Object.entries(metadataFilter).every(
|
||||
([key, value]) => noun.metadata && noun.metadata[key] === value
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Get total count before pagination
|
||||
totalCount = totalCount || filteredNouns.length
|
||||
|
||||
// Apply pagination
|
||||
const paginatedNouns = filteredNouns.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < filteredNouns.length || filteredNouns.length >= maxNouns
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedNouns.length > 0) {
|
||||
const lastItem = paginatedNouns[paginatedNouns.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
|
||||
return {
|
||||
items: paginatedNouns,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting nouns with pagination:', error)
|
||||
|
|
@ -651,122 +584,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
// If the adapter doesn't have a paginated method, fall back to the old approach
|
||||
// but with a warning and a reasonable limit
|
||||
console.warn(
|
||||
'Storage adapter does not support pagination, falling back to loading all verbs. This may cause performance issues with large datasets.'
|
||||
// Storage adapter does not support pagination
|
||||
console.error(
|
||||
'Storage adapter does not support pagination. The deprecated getAllVerbs_internal() method has been removed. Please implement getVerbsWithPagination() in your storage adapter.'
|
||||
)
|
||||
|
||||
// Get verbs with a reasonable limit to avoid memory issues
|
||||
const maxVerbs = Math.min(offset + limit + 100, 1000) // Reasonable limit
|
||||
let allVerbs: GraphVerb[] = []
|
||||
|
||||
try {
|
||||
// Try to get only the verbs we need
|
||||
allVerbs = await this.getAllVerbs()
|
||||
|
||||
// If we have too many verbs, truncate the array to avoid memory issues
|
||||
if (allVerbs.length > maxVerbs) {
|
||||
console.warn(
|
||||
`Large number of verbs (${allVerbs.length}), truncating to ${maxVerbs} for filtering`
|
||||
)
|
||||
allVerbs = allVerbs.slice(0, maxVerbs)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting all verbs:', error)
|
||||
// Return empty result on error
|
||||
return {
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
}
|
||||
|
||||
// Apply filtering if needed
|
||||
let filteredVerbs = allVerbs
|
||||
|
||||
if (options?.filter) {
|
||||
// Filter by verb type
|
||||
if (options.filter.verbType) {
|
||||
const verbTypes = Array.isArray(options.filter.verbType)
|
||||
? options.filter.verbType
|
||||
: [options.filter.verbType]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(
|
||||
(verb) => verb.type !== undefined && verbTypes.includes(verb.type)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by source ID
|
||||
if (options.filter.sourceId) {
|
||||
const sourceIds = Array.isArray(options.filter.sourceId)
|
||||
? options.filter.sourceId
|
||||
: [options.filter.sourceId]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(
|
||||
(verb) =>
|
||||
verb.sourceId !== undefined && sourceIds.includes(verb.sourceId)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by target ID
|
||||
if (options.filter.targetId) {
|
||||
const targetIds = Array.isArray(options.filter.targetId)
|
||||
? options.filter.targetId
|
||||
: [options.filter.targetId]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter(
|
||||
(verb) =>
|
||||
verb.targetId !== undefined && targetIds.includes(verb.targetId)
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by service
|
||||
if (options.filter.service) {
|
||||
const services = Array.isArray(options.filter.service)
|
||||
? options.filter.service
|
||||
: [options.filter.service]
|
||||
|
||||
filteredVerbs = filteredVerbs.filter((verb) => {
|
||||
// GraphVerb doesn't have a service property directly, check metadata
|
||||
const service = verb.metadata?.service
|
||||
return typeof service === 'string' && services.includes(service)
|
||||
})
|
||||
}
|
||||
|
||||
// Filter by metadata
|
||||
if (options.filter.metadata) {
|
||||
const metadataFilter = options.filter.metadata
|
||||
filteredVerbs = filteredVerbs.filter((verb) => {
|
||||
if (!verb.metadata) return false
|
||||
|
||||
// Check if all metadata keys match
|
||||
return Object.entries(metadataFilter).every(
|
||||
([key, value]) => verb.metadata && verb.metadata[key] === value
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Get total count before pagination
|
||||
totalCount = totalCount || filteredVerbs.length
|
||||
|
||||
// Apply pagination
|
||||
const paginatedVerbs = filteredVerbs.slice(offset, offset + limit)
|
||||
const hasMore = offset + limit < filteredVerbs.length || filteredVerbs.length >= maxVerbs
|
||||
|
||||
// Set next cursor if there are more items
|
||||
let nextCursor: string | undefined = undefined
|
||||
if (hasMore && paginatedVerbs.length > 0) {
|
||||
const lastItem = paginatedVerbs[paginatedVerbs.length - 1]
|
||||
nextCursor = lastItem.id
|
||||
}
|
||||
|
||||
|
||||
return {
|
||||
items: paginatedVerbs,
|
||||
totalCount,
|
||||
hasMore,
|
||||
nextCursor
|
||||
items: [],
|
||||
totalCount: 0,
|
||||
hasMore: false
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error getting verbs with pagination:', error)
|
||||
|
|
@ -851,12 +677,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
*/
|
||||
protected abstract getNoun_internal(id: string): Promise<HNSWNoun | null>
|
||||
|
||||
/**
|
||||
* Get all nouns from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllNouns_internal(): Promise<HNSWNoun[]>
|
||||
|
||||
/**
|
||||
* Get nouns by noun type
|
||||
* This method should be implemented by each specific adapter
|
||||
|
|
@ -883,12 +703,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
*/
|
||||
protected abstract getVerb_internal(id: string): Promise<HNSWVerb | null>
|
||||
|
||||
/**
|
||||
* Get all verbs from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
*/
|
||||
protected abstract getAllVerbs_internal(): Promise<HNSWVerb[]>
|
||||
|
||||
/**
|
||||
* Get verbs by source
|
||||
* This method should be implemented by each specific adapter
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
|
||||
import { StorageAdapter } from '../coreTypes.js'
|
||||
import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js'
|
||||
|
||||
export interface MetadataIndexEntry {
|
||||
field: string
|
||||
|
|
@ -13,6 +14,12 @@ export interface MetadataIndexEntry {
|
|||
lastUpdated: number
|
||||
}
|
||||
|
||||
export interface FieldIndexData {
|
||||
// Maps value -> count for quick filter discovery
|
||||
values: Record<string, number>
|
||||
lastUpdated: number
|
||||
}
|
||||
|
||||
export interface MetadataIndexStats {
|
||||
totalEntries: number
|
||||
totalIds: number
|
||||
|
|
@ -39,6 +46,11 @@ export class MetadataIndexManager {
|
|||
private indexCache = new Map<string, MetadataIndexEntry>()
|
||||
private dirtyEntries = new Set<string>()
|
||||
private isRebuilding = false
|
||||
private metadataCache: MetadataIndexCache
|
||||
private fieldIndexes = new Map<string, FieldIndexData>()
|
||||
private dirtyFields = new Set<string>()
|
||||
private lastFlushTime = Date.now()
|
||||
private autoFlushThreshold = 50 // Start with 50, will adapt based on usage
|
||||
|
||||
constructor(storage: StorageAdapter, config: MetadataIndexConfig = {}) {
|
||||
this.storage = storage
|
||||
|
|
@ -49,6 +61,13 @@ export class MetadataIndexManager {
|
|||
indexedFields: config.indexedFields ?? [],
|
||||
excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt', 'embedding', 'vector', 'embeddings', 'vectors']
|
||||
}
|
||||
|
||||
// Initialize metadata cache with similar config to search cache
|
||||
this.metadataCache = new MetadataIndexCache({
|
||||
maxAge: 5 * 60 * 1000, // 5 minutes
|
||||
maxSize: 500, // 500 entries (field indexes + value chunks)
|
||||
enabled: true
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -173,7 +192,7 @@ export class MetadataIndexManager {
|
|||
/**
|
||||
* Add item to metadata indexes
|
||||
*/
|
||||
async addToIndex(id: string, metadata: any): Promise<void> {
|
||||
async addToIndex(id: string, metadata: any, skipFlush: boolean = false): Promise<void> {
|
||||
const fields = this.extractIndexableFields(metadata)
|
||||
|
||||
for (const { field, value } of fields) {
|
||||
|
|
@ -196,7 +215,65 @@ export class MetadataIndexManager {
|
|||
entry.ids.add(id)
|
||||
entry.lastUpdated = Date.now()
|
||||
this.dirtyEntries.add(key)
|
||||
|
||||
// Update field index
|
||||
await this.updateFieldIndex(field, value, 1)
|
||||
}
|
||||
|
||||
// Adaptive auto-flush based on usage patterns
|
||||
if (!skipFlush) {
|
||||
const timeSinceLastFlush = Date.now() - this.lastFlushTime
|
||||
const shouldAutoFlush =
|
||||
this.dirtyEntries.size >= this.autoFlushThreshold || // Size threshold
|
||||
(this.dirtyEntries.size > 10 && timeSinceLastFlush > 5000) // Time threshold (5 seconds)
|
||||
|
||||
if (shouldAutoFlush) {
|
||||
const startTime = Date.now()
|
||||
await this.flush()
|
||||
const flushTime = Date.now() - startTime
|
||||
|
||||
// Adapt threshold based on flush performance
|
||||
if (flushTime < 50) {
|
||||
// Fast flush, can handle more entries
|
||||
this.autoFlushThreshold = Math.min(200, this.autoFlushThreshold * 1.2)
|
||||
} else if (flushTime > 200) {
|
||||
// Slow flush, reduce batch size
|
||||
this.autoFlushThreshold = Math.max(20, this.autoFlushThreshold * 0.8)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Invalidate cache for these fields
|
||||
for (const { field } of fields) {
|
||||
this.metadataCache.invalidatePattern(`field_values_${field}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update field index with value count
|
||||
*/
|
||||
private async updateFieldIndex(field: string, value: any, delta: number): Promise<void> {
|
||||
let fieldIndex = this.fieldIndexes.get(field)
|
||||
|
||||
if (!fieldIndex) {
|
||||
// Load from storage if not in memory
|
||||
fieldIndex = await this.loadFieldIndex(field) ?? {
|
||||
values: {},
|
||||
lastUpdated: Date.now()
|
||||
}
|
||||
this.fieldIndexes.set(field, fieldIndex)
|
||||
}
|
||||
|
||||
const normalizedValue = this.normalizeValue(value)
|
||||
fieldIndex.values[normalizedValue] = (fieldIndex.values[normalizedValue] || 0) + delta
|
||||
|
||||
// Remove if count drops to 0
|
||||
if (fieldIndex.values[normalizedValue] <= 0) {
|
||||
delete fieldIndex.values[normalizedValue]
|
||||
}
|
||||
|
||||
fieldIndex.lastUpdated = Date.now()
|
||||
this.dirtyFields.add(field)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -220,12 +297,18 @@ export class MetadataIndexManager {
|
|||
entry.lastUpdated = Date.now()
|
||||
this.dirtyEntries.add(key)
|
||||
|
||||
// Update field index
|
||||
await this.updateFieldIndex(field, value, -1)
|
||||
|
||||
// If no IDs left, mark for cleanup
|
||||
if (entry.ids.size === 0) {
|
||||
this.indexCache.delete(key)
|
||||
await this.deleteIndexEntry(key)
|
||||
}
|
||||
}
|
||||
|
||||
// Invalidate cache
|
||||
this.metadataCache.invalidatePattern(`field_values_${field}`)
|
||||
}
|
||||
} else {
|
||||
// Remove from all indexes (slower, requires scanning)
|
||||
|
|
@ -245,12 +328,19 @@ export class MetadataIndexManager {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get IDs for a specific field-value combination
|
||||
* Get IDs for a specific field-value combination with caching
|
||||
*/
|
||||
async getIds(field: string, value: any): Promise<string[]> {
|
||||
const key = this.getIndexKey(field, value)
|
||||
|
||||
// Try cache first
|
||||
// Check metadata cache first
|
||||
const cacheKey = `ids_${key}`
|
||||
const cachedIds = this.metadataCache.get(cacheKey)
|
||||
if (cachedIds) {
|
||||
return cachedIds
|
||||
}
|
||||
|
||||
// Try in-memory cache
|
||||
let entry = this.indexCache.get(key)
|
||||
|
||||
// Load from storage if not cached
|
||||
|
|
@ -262,7 +352,73 @@ export class MetadataIndexManager {
|
|||
}
|
||||
}
|
||||
|
||||
return entry ? Array.from(entry.ids) : []
|
||||
const ids = entry ? Array.from(entry.ids) : []
|
||||
|
||||
// Cache the result
|
||||
this.metadataCache.set(cacheKey, ids)
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all available values for a field (for filter discovery)
|
||||
*/
|
||||
async getFilterValues(field: string): Promise<string[]> {
|
||||
// Check cache first
|
||||
const cacheKey = `field_values_${field}`
|
||||
const cachedValues = this.metadataCache.get(cacheKey)
|
||||
if (cachedValues) {
|
||||
return cachedValues
|
||||
}
|
||||
|
||||
// Check in-memory field indexes first
|
||||
let fieldIndex = this.fieldIndexes.get(field)
|
||||
|
||||
// If not in memory, load from storage
|
||||
if (!fieldIndex) {
|
||||
const loaded = await this.loadFieldIndex(field)
|
||||
if (loaded) {
|
||||
fieldIndex = loaded
|
||||
this.fieldIndexes.set(field, loaded)
|
||||
}
|
||||
}
|
||||
|
||||
if (!fieldIndex) {
|
||||
return []
|
||||
}
|
||||
|
||||
const values = Object.keys(fieldIndex.values)
|
||||
|
||||
// Cache the result
|
||||
this.metadataCache.set(cacheKey, values)
|
||||
|
||||
return values
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all indexed fields (for filter discovery)
|
||||
*/
|
||||
async getFilterFields(): Promise<string[]> {
|
||||
// Check cache first
|
||||
const cacheKey = 'all_filter_fields'
|
||||
const cachedFields = this.metadataCache.get(cacheKey)
|
||||
if (cachedFields) {
|
||||
return cachedFields
|
||||
}
|
||||
|
||||
// Get fields from in-memory indexes and storage
|
||||
const fields = new Set<string>(this.fieldIndexes.keys())
|
||||
|
||||
// Also scan storage for persisted field indexes (in case not loaded)
|
||||
// This would require a new storage method to list field indexes
|
||||
// For now, just use in-memory fields
|
||||
|
||||
const fieldsArray = Array.from(fields)
|
||||
|
||||
// Cache the result
|
||||
this.metadataCache.set(cacheKey, fieldsArray)
|
||||
|
||||
return fieldsArray
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -291,6 +447,10 @@ export class MetadataIndexManager {
|
|||
case '$eq':
|
||||
criteria.push({ field: key, values: [operand] })
|
||||
break
|
||||
case '$includes':
|
||||
// For $includes, the operand is the value we're looking for in an array field
|
||||
criteria.push({ field: key, values: [operand] })
|
||||
break
|
||||
// For other operators, we can't use index efficiently, skip for now
|
||||
default:
|
||||
break
|
||||
|
|
@ -376,8 +536,13 @@ export class MetadataIndexManager {
|
|||
* Flush dirty entries to storage
|
||||
*/
|
||||
async flush(): Promise<void> {
|
||||
if (this.dirtyEntries.size === 0 && this.dirtyFields.size === 0) {
|
||||
return // Nothing to flush
|
||||
}
|
||||
|
||||
const promises: Promise<void>[] = []
|
||||
|
||||
// Flush value entries
|
||||
for (const key of this.dirtyEntries) {
|
||||
const entry = this.indexCache.get(key)
|
||||
if (entry) {
|
||||
|
|
@ -385,8 +550,69 @@ export class MetadataIndexManager {
|
|||
}
|
||||
}
|
||||
|
||||
// Flush field indexes
|
||||
for (const field of this.dirtyFields) {
|
||||
const fieldIndex = this.fieldIndexes.get(field)
|
||||
if (fieldIndex) {
|
||||
promises.push(this.saveFieldIndex(field, fieldIndex))
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(promises)
|
||||
this.dirtyEntries.clear()
|
||||
this.dirtyFields.clear()
|
||||
this.lastFlushTime = Date.now()
|
||||
}
|
||||
|
||||
/**
|
||||
* Load field index from storage
|
||||
*/
|
||||
private async loadFieldIndex(field: string): Promise<FieldIndexData | null> {
|
||||
try {
|
||||
const filename = this.getFieldIndexFilename(field)
|
||||
const cacheKey = `field_index_${filename}`
|
||||
|
||||
// Check cache first
|
||||
const cached = this.metadataCache.get(cacheKey)
|
||||
if (cached) {
|
||||
return cached
|
||||
}
|
||||
|
||||
// Load from storage
|
||||
const indexId = `__metadata_field_index__${filename}`
|
||||
const data = await this.storage.getMetadata(indexId)
|
||||
|
||||
if (data) {
|
||||
const fieldIndex = {
|
||||
values: data.values || {},
|
||||
lastUpdated: data.lastUpdated || Date.now()
|
||||
}
|
||||
|
||||
// Cache it
|
||||
this.metadataCache.set(cacheKey, fieldIndex)
|
||||
|
||||
return fieldIndex
|
||||
}
|
||||
} catch (error) {
|
||||
// Field index doesn't exist yet
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Save field index to storage
|
||||
*/
|
||||
private async saveFieldIndex(field: string, fieldIndex: FieldIndexData): Promise<void> {
|
||||
const filename = this.getFieldIndexFilename(field)
|
||||
const indexId = `__metadata_field_index__${filename}`
|
||||
|
||||
await this.storage.saveMetadata(indexId, {
|
||||
values: fieldIndex.values,
|
||||
lastUpdated: fieldIndex.lastUpdated
|
||||
})
|
||||
|
||||
// Invalidate cache
|
||||
this.metadataCache.invalidatePattern(`field_index_${filename}`)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -413,7 +639,7 @@ export class MetadataIndexManager {
|
|||
}
|
||||
|
||||
/**
|
||||
* Rebuild entire index from scratch
|
||||
* Rebuild entire index from scratch using pagination
|
||||
*/
|
||||
async rebuild(): Promise<void> {
|
||||
if (this.isRebuilding) return
|
||||
|
|
@ -423,25 +649,51 @@ export class MetadataIndexManager {
|
|||
// Clear existing indexes
|
||||
this.indexCache.clear()
|
||||
this.dirtyEntries.clear()
|
||||
this.fieldIndexes.clear()
|
||||
this.dirtyFields.clear()
|
||||
|
||||
// Get all nouns and rebuild their metadata indexes
|
||||
const nouns = await this.storage.getAllNouns()
|
||||
// Rebuild noun metadata indexes using pagination
|
||||
let nounOffset = 0
|
||||
const nounLimit = 100
|
||||
let hasMoreNouns = true
|
||||
|
||||
for (const noun of nouns) {
|
||||
const metadata = await this.storage.getMetadata(noun.id)
|
||||
if (metadata) {
|
||||
await this.addToIndex(noun.id, metadata)
|
||||
while (hasMoreNouns) {
|
||||
const result = await this.storage.getNouns({
|
||||
pagination: { offset: nounOffset, limit: nounLimit }
|
||||
})
|
||||
|
||||
for (const noun of result.items) {
|
||||
const metadata = await this.storage.getMetadata(noun.id)
|
||||
if (metadata) {
|
||||
// Skip flush during rebuild for performance
|
||||
await this.addToIndex(noun.id, metadata, true)
|
||||
}
|
||||
}
|
||||
|
||||
hasMoreNouns = result.hasMore
|
||||
nounOffset += nounLimit
|
||||
}
|
||||
|
||||
// Get all verbs and rebuild their metadata indexes
|
||||
const verbs = await this.storage.getAllVerbs()
|
||||
// Rebuild verb metadata indexes using pagination
|
||||
let verbOffset = 0
|
||||
const verbLimit = 100
|
||||
let hasMoreVerbs = true
|
||||
|
||||
for (const verb of verbs) {
|
||||
const metadata = await this.storage.getVerbMetadata(verb.id)
|
||||
if (metadata) {
|
||||
await this.addToIndex(verb.id, metadata)
|
||||
while (hasMoreVerbs) {
|
||||
const result = await this.storage.getVerbs({
|
||||
pagination: { offset: verbOffset, limit: verbLimit }
|
||||
})
|
||||
|
||||
for (const verb of result.items) {
|
||||
const metadata = await this.storage.getVerbMetadata(verb.id)
|
||||
if (metadata) {
|
||||
// Skip flush during rebuild for performance
|
||||
await this.addToIndex(verb.id, metadata, true)
|
||||
}
|
||||
}
|
||||
|
||||
hasMoreVerbs = result.hasMore
|
||||
verbOffset += verbLimit
|
||||
}
|
||||
|
||||
// Flush to storage
|
||||
|
|
|
|||
151
src/utils/metadataIndexCache.ts
Normal file
151
src/utils/metadataIndexCache.ts
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* MetadataIndexCache - Caches metadata index data for improved performance
|
||||
* Reuses the same pattern as SearchCache for consistency
|
||||
*/
|
||||
|
||||
export interface MetadataCacheEntry {
|
||||
data: any // Field index or value chunk data
|
||||
timestamp: number
|
||||
hits: number
|
||||
}
|
||||
|
||||
export interface MetadataIndexCacheConfig {
|
||||
maxAge?: number // Maximum age in milliseconds (default: 5 minutes)
|
||||
maxSize?: number // Maximum number of cached entries (default: 500)
|
||||
enabled?: boolean // Whether caching is enabled (default: true)
|
||||
hitCountWeight?: number // Weight for hit count in eviction policy (default: 0.3)
|
||||
}
|
||||
|
||||
export class MetadataIndexCache {
|
||||
private cache = new Map<string, MetadataCacheEntry>()
|
||||
private maxAge: number
|
||||
private maxSize: number
|
||||
private enabled: boolean
|
||||
private hitCountWeight: number
|
||||
|
||||
// Cache statistics
|
||||
private hits = 0
|
||||
private misses = 0
|
||||
private evictions = 0
|
||||
|
||||
constructor(config: MetadataIndexCacheConfig = {}) {
|
||||
this.maxAge = config.maxAge ?? 5 * 60 * 1000 // 5 minutes
|
||||
this.maxSize = config.maxSize ?? 500 // More entries than SearchCache since indexes are smaller
|
||||
this.enabled = config.enabled ?? true
|
||||
this.hitCountWeight = config.hitCountWeight ?? 0.3
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached entry
|
||||
*/
|
||||
get(key: string): any | undefined {
|
||||
if (!this.enabled) return undefined
|
||||
|
||||
const entry = this.cache.get(key)
|
||||
if (!entry) {
|
||||
this.misses++
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Check if entry is expired
|
||||
if (Date.now() - entry.timestamp > this.maxAge) {
|
||||
this.cache.delete(key)
|
||||
this.misses++
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Update hit count
|
||||
entry.hits++
|
||||
this.hits++
|
||||
return entry.data
|
||||
}
|
||||
|
||||
/**
|
||||
* Set cache entry
|
||||
*/
|
||||
set(key: string, data: any): void {
|
||||
if (!this.enabled) return
|
||||
|
||||
// Evict entries if at max size
|
||||
if (this.cache.size >= this.maxSize) {
|
||||
this.evictLeastValuable()
|
||||
}
|
||||
|
||||
this.cache.set(key, {
|
||||
data,
|
||||
timestamp: Date.now(),
|
||||
hits: 0
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict least valuable entry based on age and hit count
|
||||
*/
|
||||
private evictLeastValuable(): void {
|
||||
let leastValuableKey: string | null = null
|
||||
let lowestScore = Infinity
|
||||
|
||||
for (const [key, entry] of this.cache.entries()) {
|
||||
const age = Date.now() - entry.timestamp
|
||||
const ageScore = age / this.maxAge
|
||||
const hitScore = entry.hits * this.hitCountWeight
|
||||
const score = hitScore - ageScore
|
||||
|
||||
if (score < lowestScore) {
|
||||
lowestScore = score
|
||||
leastValuableKey = key
|
||||
}
|
||||
}
|
||||
|
||||
if (leastValuableKey) {
|
||||
this.cache.delete(leastValuableKey)
|
||||
this.evictions++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalidate cache entries matching a pattern
|
||||
*/
|
||||
invalidatePattern(pattern: string): void {
|
||||
const keysToDelete: string[] = []
|
||||
for (const key of this.cache.keys()) {
|
||||
if (key.includes(pattern)) {
|
||||
keysToDelete.push(key)
|
||||
}
|
||||
}
|
||||
keysToDelete.forEach(key => this.cache.delete(key))
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cache entries
|
||||
*/
|
||||
clear(): void {
|
||||
this.cache.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
size: this.cache.size,
|
||||
hits: this.hits,
|
||||
misses: this.misses,
|
||||
hitRate: this.hits / (this.hits + this.misses) || 0,
|
||||
evictions: this.evictions
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get estimated memory usage
|
||||
*/
|
||||
getMemoryUsage(): number {
|
||||
// Rough estimate: 100 bytes per entry + data size
|
||||
let totalSize = 0
|
||||
for (const entry of this.cache.values()) {
|
||||
totalSize += 100 // Base overhead
|
||||
totalSize += JSON.stringify(entry.data).length * 2 // Unicode chars
|
||||
}
|
||||
return totalSize
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue