diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index b4b90eff..b9f5e35b 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -757,88 +757,7 @@ export class MetadataIndexManager { this.dirtySparseIndices.clear() } - /** - * Split a chunk without saving immediately — returns the new chunks for deferred save. - * Used by addToChunkedIndex() to keep splits within the deferred write batch. - */ - private async splitChunkDeferred( - chunk: ChunkData, - sparseIndex: SparseIndex - ): Promise<{ chunk1: ChunkData; chunk2: ChunkData }> { - const values = Array.from(chunk.entries.keys()).sort() - const midpoint = Math.floor(values.length / 2) - - // Create two new chunks with roaring bitmaps - const entries1 = new Map() - const entries2 = new Map() - - for (let i = 0; i < values.length; i++) { - const value = values[i] - const bitmap = chunk.entries.get(value)! - - if (i < midpoint) { - entries1.set(value, new RoaringBitmap32(bitmap.toArray())) - } else { - entries2.set(value, new RoaringBitmap32(bitmap.toArray())) - } - } - - // Create chunk objects without saving (just allocate IDs and set up data) - const chunkId1 = this.chunkManager['getNextChunkId'](chunk.field) - const chunk1: ChunkData = { - chunkId: chunkId1, - field: chunk.field, - entries: entries1, - lastUpdated: Date.now() - } - - const chunkId2 = this.chunkManager['getNextChunkId'](chunk.field) - const chunk2: ChunkData = { - chunkId: chunkId2, - field: chunk.field, - entries: entries2, - lastUpdated: Date.now() - } - - // Update chunk cache (for read-after-write consistency within this operation) - this.chunkManager['chunkCache'].set(`${chunk.field}:${chunkId1}`, chunk1) - this.chunkManager['chunkCache'].set(`${chunk.field}:${chunkId2}`, chunk2) - - // Update sparse index - sparseIndex.removeChunk(chunk.chunkId) - - const descriptor1: ChunkDescriptor = { - chunkId: chunk1.chunkId, - field: chunk1.field, - valueCount: entries1.size, - idCount: Array.from(entries1.values()).reduce((sum, bitmap) => sum + bitmap.size, 0), - zoneMap: this.chunkManager.calculateZoneMap(chunk1), - lastUpdated: Date.now(), - splitThreshold: 80, - mergeThreshold: 20 - } - - const descriptor2: ChunkDescriptor = { - chunkId: chunk2.chunkId, - field: chunk2.field, - valueCount: entries2.size, - idCount: Array.from(entries2.values()).reduce((sum, bitmap) => sum + bitmap.size, 0), - zoneMap: this.chunkManager.calculateZoneMap(chunk2), - lastUpdated: Date.now(), - splitThreshold: 80, - mergeThreshold: 20 - } - - sparseIndex.registerChunk(descriptor1, this.chunkManager.createBloomFilter(chunk1)) - sparseIndex.registerChunk(descriptor2, this.chunkManager.createBloomFilter(chunk2)) - - // Delete old chunk from storage (this still writes immediately as it's a deletion) - await this.chunkManager.deleteChunk(chunk.field, chunk.chunkId) - - prodLog.debug(`Split chunk ${chunk.field}:${chunk.chunkId} into ${chunk1.chunkId} and ${chunk2.chunkId} (deferred save)`) - - return { chunk1, chunk2 } - } + // splitChunkDeferred — DELETED. Only called from addToChunkedIndex (also deleted). /** * Get IDs for a value using chunked sparse index with roaring bitmaps @@ -1047,149 +966,9 @@ export class MetadataIndexManager { return result.size > 0 ? this.idMapper.intsIterableToUuids(result) : [] } - /** - * Add value-ID mapping to chunked index - * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) - */ - private async addToChunkedIndex(field: string, value: any, id: string): Promise { - // Load or create sparse index via UnifiedCache (lazy loading) - let sparseIndex = await this.loadSparseIndex(field) - if (!sparseIndex) { - // Create new sparse index - const stats = this.fieldStats.get(field) - const chunkSize = stats - ? this.chunkingStrategy.getOptimalChunkSize({ - uniqueValues: stats.cardinality.uniqueValues, - distribution: stats.cardinality.distribution, - avgIdsPerValue: stats.cardinality.totalValues / Math.max(1, stats.cardinality.uniqueValues) - }) - : 50 + // addToChunkedIndex — DELETED. Column store handles all writes. - sparseIndex = new SparseIndex(field, chunkSize) - } - - const normalizedValue = this.normalizeValue(value, field) - - // Find existing chunk for this value (check zone maps) - const candidateChunkIds = sparseIndex.findChunksForValue(normalizedValue) - - let targetChunk: ChunkData | null = null - let targetChunkId: number | null = null - - // Try to find an existing chunk with this value - for (const chunkId of candidateChunkIds) { - const chunk = await this.chunkManager.loadChunk(field, chunkId) - if (chunk && chunk.entries.has(normalizedValue)) { - targetChunk = chunk - targetChunkId = chunkId - break - } - } - - // If no chunk has this value, find chunk with space or create new one - if (!targetChunk) { - // Find a chunk with available space - for (const chunkId of sparseIndex.getAllChunkIds()) { - const chunk = await this.chunkManager.loadChunk(field, chunkId) - const descriptor = sparseIndex.getChunk(chunkId) - if (chunk && descriptor && chunk.entries.size < descriptor.splitThreshold) { - targetChunk = chunk - targetChunkId = chunkId - break - } - } - } - - // Create new chunk if needed - if (!targetChunk) { - targetChunk = await this.chunkManager.createChunk(field) - targetChunkId = targetChunk.chunkId - - // Register in sparse index - const descriptor: ChunkDescriptor = { - chunkId: targetChunk.chunkId, - field, - valueCount: 0, - idCount: 0, - zoneMap: { min: null, max: null, count: 0, hasNulls: false }, - lastUpdated: Date.now(), - splitThreshold: 80, - mergeThreshold: 20 - } - sparseIndex.registerChunk(descriptor) - } - - // Add to chunk - await this.chunkManager.addToChunk(targetChunk, normalizedValue, id) - // Defer chunk save — mark dirty instead of writing immediately - this.dirtyChunks.set(`${targetChunk.field}:${targetChunk.chunkId}`, targetChunk) - - // Update chunk descriptor in sparse index - const updatedZoneMap = this.chunkManager.calculateZoneMap(targetChunk) - const updatedBloomFilter = this.chunkManager.createBloomFilter(targetChunk) - - sparseIndex.updateChunk(targetChunkId!, { - valueCount: targetChunk.entries.size, - idCount: Array.from(targetChunk.entries.values()).reduce((sum, bitmap) => sum + bitmap.size, 0), - zoneMap: updatedZoneMap, - lastUpdated: Date.now() - }) - - // Update bloom filter - const descriptor = sparseIndex.getChunk(targetChunkId!) - if (descriptor) { - sparseIndex.registerChunk(descriptor, updatedBloomFilter) - } - - // Check if chunk needs splitting - if (targetChunk.entries.size > 80) { - const { chunk1, chunk2 } = await this.splitChunkDeferred(targetChunk, sparseIndex) - // Mark split result chunks as dirty instead of the original - this.dirtyChunks.delete(`${targetChunk.field}:${targetChunk.chunkId}`) - this.dirtyChunks.set(`${chunk1.field}:${chunk1.chunkId}`, chunk1) - this.dirtyChunks.set(`${chunk2.field}:${chunk2.chunkId}`, chunk2) - } - - // Defer sparse index save — mark dirty instead of writing immediately - this.dirtySparseIndices.set(field, sparseIndex) - } - - /** - * Remove ID from chunked index - * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) - */ - private async removeFromChunkedIndex(field: string, value: any, id: string): Promise { - // Load sparse index via UnifiedCache (lazy loading) - const sparseIndex = await this.loadSparseIndex(field) - if (!sparseIndex) { - return // No chunked index exists - } - - const normalizedValue = this.normalizeValue(value, field) - const candidateChunkIds = sparseIndex.findChunksForValue(normalizedValue) - - for (const chunkId of candidateChunkIds) { - const chunk = await this.chunkManager.loadChunk(field, chunkId) - if (chunk && chunk.entries.has(normalizedValue)) { - await this.chunkManager.removeFromChunk(chunk, normalizedValue, id) - // Defer chunk save — mark dirty instead of writing immediately - this.dirtyChunks.set(`${chunk.field}:${chunk.chunkId}`, chunk) - - // Update sparse index - const updatedZoneMap = this.chunkManager.calculateZoneMap(chunk) - sparseIndex.updateChunk(chunkId, { - valueCount: chunk.entries.size, - idCount: Array.from(chunk.entries.values()).reduce((sum, bitmap) => sum + bitmap.size, 0), - zoneMap: updatedZoneMap, - lastUpdated: Date.now() - }) - - // Defer sparse index save — mark dirty instead of writing immediately - this.dirtySparseIndices.set(field, sparseIndex) - break - } - } - } + // removeFromChunkedIndex — DELETED. Column store handles removes via global deleted bitmap. /** * Get IDs matching a range query using zone maps @@ -1238,25 +1017,7 @@ export class MetadataIndexManager { return `field_${field}` } - /** - * Generate value chunk filename for scalable storage - */ - private getValueChunkFilename(field: string, value: any, chunkIndex: number = 0): string { - const normalizedValue = this.normalizeValue(value, field) // Pass field for bucketing! - const safeValue = this.makeSafeFilename(normalizedValue) - return `${field}_${safeValue}_chunk${chunkIndex}` - } - - /** - * Make a value safe for use in filenames - */ - private makeSafeFilename(value: string): string { - // Replace unsafe characters and limit length - return value - .replace(/[^a-zA-Z0-9-_]/g, '_') - .substring(0, 50) - .toLowerCase() - } + // getValueChunkFilename, makeSafeFilename — DELETED. Sparse index file naming no longer needed. /** * Normalize value for consistent indexing with VALUE-BASED temporal detection