David Snelling
f8c45f2d8d
Open source vector database with HNSW indexing, graph relationships, and metadata facets. Features CLI with professional augmentation registry integration for discovering extensions and capabilities.
770 lines
No EOL
31 KiB
JavaScript
770 lines
No EOL
31 KiB
JavaScript
/**
|
|
* Metadata Index System
|
|
* Maintains inverted indexes for fast metadata filtering
|
|
* Automatically updates indexes when data changes
|
|
*/
|
|
import { MetadataIndexCache } from './metadataIndexCache.js';
|
|
import { prodLog } from './logger.js';
|
|
/**
|
|
* Manages metadata indexes for fast filtering
|
|
* Maintains inverted indexes: field+value -> list of IDs
|
|
*/
|
|
export class MetadataIndexManager {
|
|
constructor(storage, config = {}) {
|
|
this.indexCache = new Map();
|
|
this.dirtyEntries = new Set();
|
|
this.isRebuilding = false;
|
|
this.fieldIndexes = new Map();
|
|
this.dirtyFields = new Set();
|
|
this.lastFlushTime = Date.now();
|
|
this.autoFlushThreshold = 10; // Start with 10 for more frequent non-blocking flushes
|
|
this.storage = storage;
|
|
this.config = {
|
|
maxIndexSize: config.maxIndexSize ?? 10000,
|
|
rebuildThreshold: config.rebuildThreshold ?? 0.1,
|
|
autoOptimize: config.autoOptimize ?? true,
|
|
indexedFields: config.indexedFields ?? [],
|
|
excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt', 'embedding', 'vector', 'embeddings', 'vectors']
|
|
};
|
|
// Initialize metadata cache with similar config to search cache
|
|
this.metadataCache = new MetadataIndexCache({
|
|
maxAge: 5 * 60 * 1000, // 5 minutes
|
|
maxSize: 500, // 500 entries (field indexes + value chunks)
|
|
enabled: true
|
|
});
|
|
}
|
|
/**
|
|
* Get index key for field and value
|
|
*/
|
|
getIndexKey(field, value) {
|
|
const normalizedValue = this.normalizeValue(value);
|
|
return `${field}:${normalizedValue}`;
|
|
}
|
|
/**
|
|
* Generate field index filename for filter discovery
|
|
*/
|
|
getFieldIndexFilename(field) {
|
|
return `field_${field}`;
|
|
}
|
|
/**
|
|
* Generate value chunk filename for scalable storage
|
|
*/
|
|
getValueChunkFilename(field, value, chunkIndex = 0) {
|
|
const normalizedValue = this.normalizeValue(value);
|
|
const safeValue = this.makeSafeFilename(normalizedValue);
|
|
return `${field}_${safeValue}_chunk${chunkIndex}`;
|
|
}
|
|
/**
|
|
* Make a value safe for use in filenames
|
|
*/
|
|
makeSafeFilename(value) {
|
|
// Replace unsafe characters and limit length
|
|
return value
|
|
.replace(/[^a-zA-Z0-9-_]/g, '_')
|
|
.substring(0, 50)
|
|
.toLowerCase();
|
|
}
|
|
/**
|
|
* Normalize value for consistent indexing
|
|
*/
|
|
normalizeValue(value) {
|
|
if (value === null || value === undefined)
|
|
return '__NULL__';
|
|
if (typeof value === 'boolean')
|
|
return value ? '__TRUE__' : '__FALSE__';
|
|
if (typeof value === 'number')
|
|
return value.toString();
|
|
if (Array.isArray(value)) {
|
|
const joined = value.map(v => this.normalizeValue(v)).join(',');
|
|
// Hash very long array values to avoid filesystem limits
|
|
if (joined.length > 100) {
|
|
return this.hashValue(joined);
|
|
}
|
|
return joined;
|
|
}
|
|
const stringValue = String(value).toLowerCase().trim();
|
|
// Hash very long string values to avoid filesystem limits
|
|
if (stringValue.length > 100) {
|
|
return this.hashValue(stringValue);
|
|
}
|
|
return stringValue;
|
|
}
|
|
/**
|
|
* Create a short hash for long values to avoid filesystem filename limits
|
|
*/
|
|
hashValue(value) {
|
|
// Simple hash function to create shorter keys
|
|
let hash = 0;
|
|
for (let i = 0; i < value.length; i++) {
|
|
const char = value.charCodeAt(i);
|
|
hash = ((hash << 5) - hash) + char;
|
|
hash = hash & hash; // Convert to 32-bit integer
|
|
}
|
|
return `__HASH_${Math.abs(hash).toString(36)}`;
|
|
}
|
|
/**
|
|
* Check if field should be indexed
|
|
*/
|
|
shouldIndexField(field) {
|
|
if (this.config.excludeFields.includes(field))
|
|
return false;
|
|
if (this.config.indexedFields.length > 0) {
|
|
return this.config.indexedFields.includes(field);
|
|
}
|
|
return true;
|
|
}
|
|
/**
|
|
* Extract indexable field-value pairs from metadata
|
|
*/
|
|
extractIndexableFields(metadata) {
|
|
const fields = [];
|
|
const extract = (obj, prefix = '') => {
|
|
for (const [key, value] of Object.entries(obj)) {
|
|
const fullKey = prefix ? `${prefix}.${key}` : key;
|
|
if (!this.shouldIndexField(fullKey))
|
|
continue;
|
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
// Recurse into nested objects
|
|
extract(value, fullKey);
|
|
}
|
|
else {
|
|
// Index this field
|
|
fields.push({ field: fullKey, value });
|
|
// If it's an array, also index each element
|
|
if (Array.isArray(value)) {
|
|
for (const item of value) {
|
|
fields.push({ field: fullKey, value: item });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
if (metadata && typeof metadata === 'object') {
|
|
extract(metadata);
|
|
}
|
|
return fields;
|
|
}
|
|
/**
|
|
* Add item to metadata indexes
|
|
*/
|
|
async addToIndex(id, metadata, skipFlush = false) {
|
|
const fields = this.extractIndexableFields(metadata);
|
|
for (let i = 0; i < fields.length; i++) {
|
|
const { field, value } = fields[i];
|
|
const key = this.getIndexKey(field, value);
|
|
// Get or create index entry
|
|
let entry = this.indexCache.get(key);
|
|
if (!entry) {
|
|
const loadedEntry = await this.loadIndexEntry(key);
|
|
entry = loadedEntry ?? {
|
|
field,
|
|
value: this.normalizeValue(value),
|
|
ids: new Set(),
|
|
lastUpdated: Date.now()
|
|
};
|
|
this.indexCache.set(key, entry);
|
|
}
|
|
// Add ID to entry
|
|
entry.ids.add(id);
|
|
entry.lastUpdated = Date.now();
|
|
this.dirtyEntries.add(key);
|
|
// Update field index
|
|
await this.updateFieldIndex(field, value, 1);
|
|
// Yield to event loop every 5 fields to prevent blocking
|
|
if (i % 5 === 4) {
|
|
await this.yieldToEventLoop();
|
|
}
|
|
}
|
|
// Adaptive auto-flush based on usage patterns
|
|
if (!skipFlush) {
|
|
const timeSinceLastFlush = Date.now() - this.lastFlushTime;
|
|
const shouldAutoFlush = this.dirtyEntries.size >= this.autoFlushThreshold || // Size threshold
|
|
(this.dirtyEntries.size > 10 && timeSinceLastFlush > 5000); // Time threshold (5 seconds)
|
|
if (shouldAutoFlush) {
|
|
const startTime = Date.now();
|
|
await this.flush();
|
|
const flushTime = Date.now() - startTime;
|
|
// Adapt threshold based on flush performance
|
|
if (flushTime < 50) {
|
|
// Fast flush, can handle more entries
|
|
this.autoFlushThreshold = Math.min(200, this.autoFlushThreshold * 1.2);
|
|
}
|
|
else if (flushTime > 200) {
|
|
// Slow flush, reduce batch size
|
|
this.autoFlushThreshold = Math.max(20, this.autoFlushThreshold * 0.8);
|
|
}
|
|
// Yield to event loop after flush to prevent blocking
|
|
await this.yieldToEventLoop();
|
|
}
|
|
}
|
|
// Invalidate cache for these fields
|
|
for (const { field } of fields) {
|
|
this.metadataCache.invalidatePattern(`field_values_${field}`);
|
|
}
|
|
}
|
|
/**
|
|
* Update field index with value count
|
|
*/
|
|
async updateFieldIndex(field, value, delta) {
|
|
let fieldIndex = this.fieldIndexes.get(field);
|
|
if (!fieldIndex) {
|
|
// Load from storage if not in memory
|
|
fieldIndex = await this.loadFieldIndex(field) ?? {
|
|
values: {},
|
|
lastUpdated: Date.now()
|
|
};
|
|
this.fieldIndexes.set(field, fieldIndex);
|
|
}
|
|
const normalizedValue = this.normalizeValue(value);
|
|
fieldIndex.values[normalizedValue] = (fieldIndex.values[normalizedValue] || 0) + delta;
|
|
// Remove if count drops to 0
|
|
if (fieldIndex.values[normalizedValue] <= 0) {
|
|
delete fieldIndex.values[normalizedValue];
|
|
}
|
|
fieldIndex.lastUpdated = Date.now();
|
|
this.dirtyFields.add(field);
|
|
}
|
|
/**
|
|
* Remove item from metadata indexes
|
|
*/
|
|
async removeFromIndex(id, metadata) {
|
|
if (metadata) {
|
|
// Remove from specific field indexes
|
|
const fields = this.extractIndexableFields(metadata);
|
|
for (const { field, value } of fields) {
|
|
const key = this.getIndexKey(field, value);
|
|
let entry = this.indexCache.get(key);
|
|
if (!entry) {
|
|
const loadedEntry = await this.loadIndexEntry(key);
|
|
entry = loadedEntry ?? undefined;
|
|
}
|
|
if (entry) {
|
|
entry.ids.delete(id);
|
|
entry.lastUpdated = Date.now();
|
|
this.dirtyEntries.add(key);
|
|
// Update field index
|
|
await this.updateFieldIndex(field, value, -1);
|
|
// If no IDs left, mark for cleanup
|
|
if (entry.ids.size === 0) {
|
|
this.indexCache.delete(key);
|
|
await this.deleteIndexEntry(key);
|
|
}
|
|
}
|
|
// Invalidate cache
|
|
this.metadataCache.invalidatePattern(`field_values_${field}`);
|
|
}
|
|
}
|
|
else {
|
|
// Remove from all indexes (slower, requires scanning)
|
|
for (const [key, entry] of this.indexCache.entries()) {
|
|
if (entry.ids.has(id)) {
|
|
entry.ids.delete(id);
|
|
entry.lastUpdated = Date.now();
|
|
this.dirtyEntries.add(key);
|
|
if (entry.ids.size === 0) {
|
|
this.indexCache.delete(key);
|
|
await this.deleteIndexEntry(key);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
/**
|
|
* Get IDs for a specific field-value combination with caching
|
|
*/
|
|
async getIds(field, value) {
|
|
const key = this.getIndexKey(field, value);
|
|
// Check metadata cache first
|
|
const cacheKey = `ids_${key}`;
|
|
const cachedIds = this.metadataCache.get(cacheKey);
|
|
if (cachedIds) {
|
|
return cachedIds;
|
|
}
|
|
// Try in-memory cache
|
|
let entry = this.indexCache.get(key);
|
|
// Load from storage if not cached
|
|
if (!entry) {
|
|
const loadedEntry = await this.loadIndexEntry(key);
|
|
if (loadedEntry) {
|
|
entry = loadedEntry;
|
|
this.indexCache.set(key, entry);
|
|
}
|
|
}
|
|
const ids = entry ? Array.from(entry.ids) : [];
|
|
// Cache the result
|
|
this.metadataCache.set(cacheKey, ids);
|
|
return ids;
|
|
}
|
|
/**
|
|
* Get all available values for a field (for filter discovery)
|
|
*/
|
|
async getFilterValues(field) {
|
|
// Check cache first
|
|
const cacheKey = `field_values_${field}`;
|
|
const cachedValues = this.metadataCache.get(cacheKey);
|
|
if (cachedValues) {
|
|
return cachedValues;
|
|
}
|
|
// Check in-memory field indexes first
|
|
let fieldIndex = this.fieldIndexes.get(field);
|
|
// If not in memory, load from storage
|
|
if (!fieldIndex) {
|
|
const loaded = await this.loadFieldIndex(field);
|
|
if (loaded) {
|
|
fieldIndex = loaded;
|
|
this.fieldIndexes.set(field, loaded);
|
|
}
|
|
}
|
|
if (!fieldIndex) {
|
|
return [];
|
|
}
|
|
const values = Object.keys(fieldIndex.values);
|
|
// Cache the result
|
|
this.metadataCache.set(cacheKey, values);
|
|
return values;
|
|
}
|
|
/**
|
|
* Get all indexed fields (for filter discovery)
|
|
*/
|
|
async getFilterFields() {
|
|
// Check cache first
|
|
const cacheKey = 'all_filter_fields';
|
|
const cachedFields = this.metadataCache.get(cacheKey);
|
|
if (cachedFields) {
|
|
return cachedFields;
|
|
}
|
|
// Get fields from in-memory indexes and storage
|
|
const fields = new Set(this.fieldIndexes.keys());
|
|
// Also scan storage for persisted field indexes (in case not loaded)
|
|
// This would require a new storage method to list field indexes
|
|
// For now, just use in-memory fields
|
|
const fieldsArray = Array.from(fields);
|
|
// Cache the result
|
|
this.metadataCache.set(cacheKey, fieldsArray);
|
|
return fieldsArray;
|
|
}
|
|
/**
|
|
* Convert MongoDB-style filter to simple field-value criteria for indexing
|
|
*/
|
|
convertFilterToCriteria(filter) {
|
|
const criteria = [];
|
|
if (!filter || typeof filter !== 'object') {
|
|
return criteria;
|
|
}
|
|
for (const [key, value] of Object.entries(filter)) {
|
|
// Skip logical operators for now - handle them separately
|
|
if (key.startsWith('$'))
|
|
continue;
|
|
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
// Handle MongoDB operators
|
|
for (const [op, operand] of Object.entries(value)) {
|
|
switch (op) {
|
|
case '$in':
|
|
if (Array.isArray(operand)) {
|
|
criteria.push({ field: key, values: operand });
|
|
}
|
|
break;
|
|
case '$eq':
|
|
criteria.push({ field: key, values: [operand] });
|
|
break;
|
|
case '$includes':
|
|
// For $includes, the operand is the value we're looking for in an array field
|
|
criteria.push({ field: key, values: [operand] });
|
|
break;
|
|
// For other operators, we can't use index efficiently, skip for now
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
// Direct value or array
|
|
const values = Array.isArray(value) ? value : [value];
|
|
criteria.push({ field: key, values });
|
|
}
|
|
}
|
|
return criteria;
|
|
}
|
|
/**
|
|
* Get IDs matching MongoDB-style metadata filter using indexes where possible
|
|
*/
|
|
async getIdsForFilter(filter) {
|
|
if (!filter || Object.keys(filter).length === 0) {
|
|
return [];
|
|
}
|
|
// Handle logical operators
|
|
if (filter.$and && Array.isArray(filter.$and)) {
|
|
// For $and, we need intersection of all sub-filters
|
|
const allIds = [];
|
|
for (const subFilter of filter.$and) {
|
|
const subIds = await this.getIdsForFilter(subFilter);
|
|
allIds.push(subIds);
|
|
}
|
|
if (allIds.length === 0)
|
|
return [];
|
|
if (allIds.length === 1)
|
|
return allIds[0];
|
|
// Intersection of all sets
|
|
return allIds.reduce((intersection, currentSet) => intersection.filter(id => currentSet.includes(id)));
|
|
}
|
|
if (filter.$or && Array.isArray(filter.$or)) {
|
|
// For $or, we need union of all sub-filters
|
|
const unionIds = new Set();
|
|
for (const subFilter of filter.$or) {
|
|
const subIds = await this.getIdsForFilter(subFilter);
|
|
subIds.forEach(id => unionIds.add(id));
|
|
}
|
|
return Array.from(unionIds);
|
|
}
|
|
// Handle regular field filters
|
|
const criteria = this.convertFilterToCriteria(filter);
|
|
const idSets = [];
|
|
for (const { field, values } of criteria) {
|
|
const unionIds = new Set();
|
|
for (const value of values) {
|
|
const ids = await this.getIds(field, value);
|
|
ids.forEach(id => unionIds.add(id));
|
|
}
|
|
idSets.push(Array.from(unionIds));
|
|
}
|
|
if (idSets.length === 0)
|
|
return [];
|
|
if (idSets.length === 1)
|
|
return idSets[0];
|
|
// Intersection of all field criteria (implicit $and)
|
|
return idSets.reduce((intersection, currentSet) => intersection.filter(id => currentSet.includes(id)));
|
|
}
|
|
/**
|
|
* Get IDs matching multiple criteria (intersection) - LEGACY METHOD
|
|
* @deprecated Use getIdsForFilter instead
|
|
*/
|
|
async getIdsForCriteria(criteria) {
|
|
return this.getIdsForFilter(criteria);
|
|
}
|
|
/**
|
|
* Flush dirty entries to storage (non-blocking version)
|
|
*/
|
|
async flush() {
|
|
if (this.dirtyEntries.size === 0 && this.dirtyFields.size === 0) {
|
|
return; // Nothing to flush
|
|
}
|
|
// Process in smaller batches to avoid blocking
|
|
const BATCH_SIZE = 20;
|
|
const allPromises = [];
|
|
// Flush value entries in batches
|
|
const dirtyEntriesArray = Array.from(this.dirtyEntries);
|
|
for (let i = 0; i < dirtyEntriesArray.length; i += BATCH_SIZE) {
|
|
const batch = dirtyEntriesArray.slice(i, i + BATCH_SIZE);
|
|
const batchPromises = batch.map(key => {
|
|
const entry = this.indexCache.get(key);
|
|
return entry ? this.saveIndexEntry(key, entry) : Promise.resolve();
|
|
});
|
|
allPromises.push(...batchPromises);
|
|
// Yield to event loop between batches
|
|
if (i + BATCH_SIZE < dirtyEntriesArray.length) {
|
|
await this.yieldToEventLoop();
|
|
}
|
|
}
|
|
// Flush field indexes in batches
|
|
const dirtyFieldsArray = Array.from(this.dirtyFields);
|
|
for (let i = 0; i < dirtyFieldsArray.length; i += BATCH_SIZE) {
|
|
const batch = dirtyFieldsArray.slice(i, i + BATCH_SIZE);
|
|
const batchPromises = batch.map(field => {
|
|
const fieldIndex = this.fieldIndexes.get(field);
|
|
return fieldIndex ? this.saveFieldIndex(field, fieldIndex) : Promise.resolve();
|
|
});
|
|
allPromises.push(...batchPromises);
|
|
// Yield to event loop between batches
|
|
if (i + BATCH_SIZE < dirtyFieldsArray.length) {
|
|
await this.yieldToEventLoop();
|
|
}
|
|
}
|
|
// Wait for all operations to complete
|
|
await Promise.all(allPromises);
|
|
this.dirtyEntries.clear();
|
|
this.dirtyFields.clear();
|
|
this.lastFlushTime = Date.now();
|
|
}
|
|
/**
|
|
* Yield control back to the Node.js event loop
|
|
* Prevents blocking during long-running operations
|
|
*/
|
|
async yieldToEventLoop() {
|
|
return new Promise(resolve => setImmediate(resolve));
|
|
}
|
|
/**
|
|
* Load field index from storage
|
|
*/
|
|
async loadFieldIndex(field) {
|
|
try {
|
|
const filename = this.getFieldIndexFilename(field);
|
|
const cacheKey = `field_index_${filename}`;
|
|
// Check cache first
|
|
const cached = this.metadataCache.get(cacheKey);
|
|
if (cached) {
|
|
return cached;
|
|
}
|
|
// Load from storage
|
|
const indexId = `__metadata_field_index__${filename}`;
|
|
const data = await this.storage.getMetadata(indexId);
|
|
if (data) {
|
|
const fieldIndex = {
|
|
values: data.values || {},
|
|
lastUpdated: data.lastUpdated || Date.now()
|
|
};
|
|
// Cache it
|
|
this.metadataCache.set(cacheKey, fieldIndex);
|
|
return fieldIndex;
|
|
}
|
|
}
|
|
catch (error) {
|
|
// Field index doesn't exist yet
|
|
}
|
|
return null;
|
|
}
|
|
/**
|
|
* Save field index to storage
|
|
*/
|
|
async saveFieldIndex(field, fieldIndex) {
|
|
const filename = this.getFieldIndexFilename(field);
|
|
const indexId = `__metadata_field_index__${filename}`;
|
|
await this.storage.saveMetadata(indexId, {
|
|
values: fieldIndex.values,
|
|
lastUpdated: fieldIndex.lastUpdated
|
|
});
|
|
// Invalidate cache
|
|
this.metadataCache.invalidatePattern(`field_index_${filename}`);
|
|
}
|
|
/**
|
|
* Get index statistics
|
|
*/
|
|
async getStats() {
|
|
const fields = new Set();
|
|
let totalEntries = 0;
|
|
let totalIds = 0;
|
|
for (const entry of this.indexCache.values()) {
|
|
fields.add(entry.field);
|
|
totalEntries++;
|
|
totalIds += entry.ids.size;
|
|
}
|
|
return {
|
|
totalEntries,
|
|
totalIds,
|
|
fieldsIndexed: Array.from(fields),
|
|
lastRebuild: 0, // TODO: track rebuild timestamp
|
|
indexSize: totalEntries * 100 // rough estimate
|
|
};
|
|
}
|
|
/**
|
|
* Rebuild entire index from scratch using pagination
|
|
* Non-blocking version that yields control back to event loop
|
|
*/
|
|
async rebuild() {
|
|
if (this.isRebuilding)
|
|
return;
|
|
this.isRebuilding = true;
|
|
try {
|
|
prodLog.info('🔄 Starting non-blocking metadata index rebuild with batch processing to prevent socket exhaustion...');
|
|
prodLog.info(`📊 Storage adapter: ${this.storage.constructor.name}`);
|
|
prodLog.info(`🔧 Batch processing available: ${!!this.storage.getMetadataBatch}`);
|
|
// Clear existing indexes
|
|
this.indexCache.clear();
|
|
this.dirtyEntries.clear();
|
|
this.fieldIndexes.clear();
|
|
this.dirtyFields.clear();
|
|
// Rebuild noun metadata indexes using pagination
|
|
let nounOffset = 0;
|
|
const nounLimit = 25; // Even smaller batches during initialization to prevent socket exhaustion
|
|
let hasMoreNouns = true;
|
|
let totalNounsProcessed = 0;
|
|
while (hasMoreNouns) {
|
|
const result = await this.storage.getNouns({
|
|
pagination: { offset: nounOffset, limit: nounLimit }
|
|
});
|
|
// CRITICAL FIX: Use batch metadata reading to prevent socket exhaustion
|
|
const nounIds = result.items.map(noun => noun.id);
|
|
let metadataBatch;
|
|
if (this.storage.getMetadataBatch) {
|
|
// Use batch reading if available (prevents socket exhaustion)
|
|
prodLog.info(`📦 Processing metadata batch ${Math.floor(totalNounsProcessed / nounLimit) + 1} (${nounIds.length} items)...`);
|
|
metadataBatch = await this.storage.getMetadataBatch(nounIds);
|
|
const successRate = ((metadataBatch.size / nounIds.length) * 100).toFixed(1);
|
|
prodLog.info(`✅ Batch loaded ${metadataBatch.size}/${nounIds.length} metadata objects (${successRate}% success)`);
|
|
}
|
|
else {
|
|
// Fallback to individual calls with strict concurrency control
|
|
prodLog.warn(`⚠️ FALLBACK: Storage adapter missing getMetadataBatch - using individual calls with concurrency limit`);
|
|
metadataBatch = new Map();
|
|
const CONCURRENCY_LIMIT = 3; // Very conservative limit
|
|
for (let i = 0; i < nounIds.length; i += CONCURRENCY_LIMIT) {
|
|
const batch = nounIds.slice(i, i + CONCURRENCY_LIMIT);
|
|
const batchPromises = batch.map(async (id) => {
|
|
try {
|
|
const metadata = await this.storage.getMetadata(id);
|
|
return { id, metadata };
|
|
}
|
|
catch (error) {
|
|
prodLog.debug(`Failed to read metadata for ${id}:`, error);
|
|
return { id, metadata: null };
|
|
}
|
|
});
|
|
const batchResults = await Promise.all(batchPromises);
|
|
for (const { id, metadata } of batchResults) {
|
|
if (metadata) {
|
|
metadataBatch.set(id, metadata);
|
|
}
|
|
}
|
|
// Yield between batches to prevent socket exhaustion
|
|
await this.yieldToEventLoop();
|
|
}
|
|
}
|
|
// Process the metadata batch
|
|
for (const noun of result.items) {
|
|
const metadata = metadataBatch.get(noun.id);
|
|
if (metadata) {
|
|
// Skip flush during rebuild for performance
|
|
await this.addToIndex(noun.id, metadata, true);
|
|
}
|
|
}
|
|
// Yield after processing the entire batch
|
|
await this.yieldToEventLoop();
|
|
totalNounsProcessed += result.items.length;
|
|
hasMoreNouns = result.hasMore;
|
|
nounOffset += nounLimit;
|
|
// Progress logging and event loop yield after each batch
|
|
if (totalNounsProcessed % 100 === 0 || !hasMoreNouns) {
|
|
prodLog.debug(`📊 Indexed ${totalNounsProcessed} nouns...`);
|
|
}
|
|
await this.yieldToEventLoop();
|
|
}
|
|
// Rebuild verb metadata indexes using pagination
|
|
let verbOffset = 0;
|
|
const verbLimit = 25; // Even smaller batches during initialization to prevent socket exhaustion
|
|
let hasMoreVerbs = true;
|
|
let totalVerbsProcessed = 0;
|
|
while (hasMoreVerbs) {
|
|
const result = await this.storage.getVerbs({
|
|
pagination: { offset: verbOffset, limit: verbLimit }
|
|
});
|
|
// CRITICAL FIX: Use batch verb metadata reading to prevent socket exhaustion
|
|
const verbIds = result.items.map(verb => verb.id);
|
|
let verbMetadataBatch;
|
|
if (this.storage.getVerbMetadataBatch) {
|
|
// Use batch reading if available (prevents socket exhaustion)
|
|
verbMetadataBatch = await this.storage.getVerbMetadataBatch(verbIds);
|
|
prodLog.debug(`📦 Batch loaded ${verbMetadataBatch.size}/${verbIds.length} verb metadata objects`);
|
|
}
|
|
else {
|
|
// Fallback to individual calls with strict concurrency control
|
|
verbMetadataBatch = new Map();
|
|
const CONCURRENCY_LIMIT = 3; // Very conservative limit to prevent socket exhaustion
|
|
for (let i = 0; i < verbIds.length; i += CONCURRENCY_LIMIT) {
|
|
const batch = verbIds.slice(i, i + CONCURRENCY_LIMIT);
|
|
const batchPromises = batch.map(async (id) => {
|
|
try {
|
|
const metadata = await this.storage.getVerbMetadata(id);
|
|
return { id, metadata };
|
|
}
|
|
catch (error) {
|
|
prodLog.debug(`Failed to read verb metadata for ${id}:`, error);
|
|
return { id, metadata: null };
|
|
}
|
|
});
|
|
const batchResults = await Promise.all(batchPromises);
|
|
for (const { id, metadata } of batchResults) {
|
|
if (metadata) {
|
|
verbMetadataBatch.set(id, metadata);
|
|
}
|
|
}
|
|
// Yield between batches to prevent socket exhaustion
|
|
await this.yieldToEventLoop();
|
|
}
|
|
}
|
|
// Process the verb metadata batch
|
|
for (const verb of result.items) {
|
|
const metadata = verbMetadataBatch.get(verb.id);
|
|
if (metadata) {
|
|
// Skip flush during rebuild for performance
|
|
await this.addToIndex(verb.id, metadata, true);
|
|
}
|
|
}
|
|
// Yield after processing the entire batch
|
|
await this.yieldToEventLoop();
|
|
totalVerbsProcessed += result.items.length;
|
|
hasMoreVerbs = result.hasMore;
|
|
verbOffset += verbLimit;
|
|
// Progress logging and event loop yield after each batch
|
|
if (totalVerbsProcessed % 100 === 0 || !hasMoreVerbs) {
|
|
prodLog.debug(`🔗 Indexed ${totalVerbsProcessed} verbs...`);
|
|
}
|
|
await this.yieldToEventLoop();
|
|
}
|
|
// Flush to storage with final yield
|
|
prodLog.debug('💾 Flushing metadata index to storage...');
|
|
await this.flush();
|
|
await this.yieldToEventLoop();
|
|
prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`);
|
|
prodLog.info(`🎯 Initial indexing may show minor socket timeouts - this is expected and doesn't affect data processing`);
|
|
}
|
|
finally {
|
|
this.isRebuilding = false;
|
|
}
|
|
}
|
|
/**
|
|
* Load index entry from storage using safe filenames
|
|
*/
|
|
async loadIndexEntry(key) {
|
|
try {
|
|
// Extract field and value from key
|
|
const [field, value] = key.split(':', 2);
|
|
const filename = this.getValueChunkFilename(field, value);
|
|
// Load from metadata indexes directory with safe filename
|
|
const indexId = `__metadata_index__${filename}`;
|
|
const data = await this.storage.getMetadata(indexId);
|
|
if (data) {
|
|
return {
|
|
field: data.field,
|
|
value: data.value,
|
|
ids: new Set(data.ids || []),
|
|
lastUpdated: data.lastUpdated || Date.now()
|
|
};
|
|
}
|
|
}
|
|
catch (error) {
|
|
// Index entry doesn't exist yet
|
|
}
|
|
return null;
|
|
}
|
|
/**
|
|
* Save index entry to storage using safe filenames
|
|
*/
|
|
async saveIndexEntry(key, entry) {
|
|
const data = {
|
|
field: entry.field,
|
|
value: entry.value,
|
|
ids: Array.from(entry.ids),
|
|
lastUpdated: entry.lastUpdated
|
|
};
|
|
// Extract field and value from key for safe filename generation
|
|
const [field, value] = key.split(':', 2);
|
|
const filename = this.getValueChunkFilename(field, value);
|
|
// Store metadata indexes with safe filename
|
|
const indexId = `__metadata_index__${filename}`;
|
|
await this.storage.saveMetadata(indexId, data);
|
|
}
|
|
/**
|
|
* Delete index entry from storage using safe filenames
|
|
*/
|
|
async deleteIndexEntry(key) {
|
|
try {
|
|
const [field, value] = key.split(':', 2);
|
|
const filename = this.getValueChunkFilename(field, value);
|
|
const indexId = `__metadata_index__${filename}`;
|
|
await this.storage.saveMetadata(indexId, null);
|
|
}
|
|
catch (error) {
|
|
// Entry might not exist
|
|
}
|
|
}
|
|
}
|
|
//# sourceMappingURL=metadataIndex.js.map
|