refactor: delete dead sparse index write path
Column store handles all writes and primary queries. Sparse index write methods are unreachable: Deleted: addToChunkedIndex (~103 lines), removeFromChunkedIndex (~37 lines), splitChunkDeferred (~82 lines), getValueChunkFilename + makeSafeFilename (~20 lines). Total: ~240 lines of dead write-path code. Sparse index read methods kept as migration fallback for existing workspaces. Will be deleted once lazy migration (buildFromStorage) ships.
This commit is contained in:
parent
46583f2d9b
commit
11be039ae8
1 changed files with 4 additions and 243 deletions
|
|
@ -757,88 +757,7 @@ export class MetadataIndexManager {
|
||||||
this.dirtySparseIndices.clear()
|
this.dirtySparseIndices.clear()
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
// splitChunkDeferred — DELETED. Only called from addToChunkedIndex (also deleted).
|
||||||
* Split a chunk without saving immediately — returns the new chunks for deferred save.
|
|
||||||
* Used by addToChunkedIndex() to keep splits within the deferred write batch.
|
|
||||||
*/
|
|
||||||
private async splitChunkDeferred(
|
|
||||||
chunk: ChunkData,
|
|
||||||
sparseIndex: SparseIndex
|
|
||||||
): Promise<{ chunk1: ChunkData; chunk2: ChunkData }> {
|
|
||||||
const values = Array.from(chunk.entries.keys()).sort()
|
|
||||||
const midpoint = Math.floor(values.length / 2)
|
|
||||||
|
|
||||||
// Create two new chunks with roaring bitmaps
|
|
||||||
const entries1 = new Map<string, RoaringBitmap32>()
|
|
||||||
const entries2 = new Map<string, RoaringBitmap32>()
|
|
||||||
|
|
||||||
for (let i = 0; i < values.length; i++) {
|
|
||||||
const value = values[i]
|
|
||||||
const bitmap = chunk.entries.get(value)!
|
|
||||||
|
|
||||||
if (i < midpoint) {
|
|
||||||
entries1.set(value, new RoaringBitmap32(bitmap.toArray()))
|
|
||||||
} else {
|
|
||||||
entries2.set(value, new RoaringBitmap32(bitmap.toArray()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create chunk objects without saving (just allocate IDs and set up data)
|
|
||||||
const chunkId1 = this.chunkManager['getNextChunkId'](chunk.field)
|
|
||||||
const chunk1: ChunkData = {
|
|
||||||
chunkId: chunkId1,
|
|
||||||
field: chunk.field,
|
|
||||||
entries: entries1,
|
|
||||||
lastUpdated: Date.now()
|
|
||||||
}
|
|
||||||
|
|
||||||
const chunkId2 = this.chunkManager['getNextChunkId'](chunk.field)
|
|
||||||
const chunk2: ChunkData = {
|
|
||||||
chunkId: chunkId2,
|
|
||||||
field: chunk.field,
|
|
||||||
entries: entries2,
|
|
||||||
lastUpdated: Date.now()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update chunk cache (for read-after-write consistency within this operation)
|
|
||||||
this.chunkManager['chunkCache'].set(`${chunk.field}:${chunkId1}`, chunk1)
|
|
||||||
this.chunkManager['chunkCache'].set(`${chunk.field}:${chunkId2}`, chunk2)
|
|
||||||
|
|
||||||
// Update sparse index
|
|
||||||
sparseIndex.removeChunk(chunk.chunkId)
|
|
||||||
|
|
||||||
const descriptor1: ChunkDescriptor = {
|
|
||||||
chunkId: chunk1.chunkId,
|
|
||||||
field: chunk1.field,
|
|
||||||
valueCount: entries1.size,
|
|
||||||
idCount: Array.from(entries1.values()).reduce((sum, bitmap) => sum + bitmap.size, 0),
|
|
||||||
zoneMap: this.chunkManager.calculateZoneMap(chunk1),
|
|
||||||
lastUpdated: Date.now(),
|
|
||||||
splitThreshold: 80,
|
|
||||||
mergeThreshold: 20
|
|
||||||
}
|
|
||||||
|
|
||||||
const descriptor2: ChunkDescriptor = {
|
|
||||||
chunkId: chunk2.chunkId,
|
|
||||||
field: chunk2.field,
|
|
||||||
valueCount: entries2.size,
|
|
||||||
idCount: Array.from(entries2.values()).reduce((sum, bitmap) => sum + bitmap.size, 0),
|
|
||||||
zoneMap: this.chunkManager.calculateZoneMap(chunk2),
|
|
||||||
lastUpdated: Date.now(),
|
|
||||||
splitThreshold: 80,
|
|
||||||
mergeThreshold: 20
|
|
||||||
}
|
|
||||||
|
|
||||||
sparseIndex.registerChunk(descriptor1, this.chunkManager.createBloomFilter(chunk1))
|
|
||||||
sparseIndex.registerChunk(descriptor2, this.chunkManager.createBloomFilter(chunk2))
|
|
||||||
|
|
||||||
// Delete old chunk from storage (this still writes immediately as it's a deletion)
|
|
||||||
await this.chunkManager.deleteChunk(chunk.field, chunk.chunkId)
|
|
||||||
|
|
||||||
prodLog.debug(`Split chunk ${chunk.field}:${chunk.chunkId} into ${chunk1.chunkId} and ${chunk2.chunkId} (deferred save)`)
|
|
||||||
|
|
||||||
return { chunk1, chunk2 }
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get IDs for a value using chunked sparse index with roaring bitmaps
|
* Get IDs for a value using chunked sparse index with roaring bitmaps
|
||||||
|
|
@ -1047,149 +966,9 @@ export class MetadataIndexManager {
|
||||||
return result.size > 0 ? this.idMapper.intsIterableToUuids(result) : []
|
return result.size > 0 ? this.idMapper.intsIterableToUuids(result) : []
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
// addToChunkedIndex — DELETED. Column store handles all writes.
|
||||||
* Add value-ID mapping to chunked index
|
|
||||||
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
|
|
||||||
*/
|
|
||||||
private async addToChunkedIndex(field: string, value: any, id: string): Promise<void> {
|
|
||||||
// Load or create sparse index via UnifiedCache (lazy loading)
|
|
||||||
let sparseIndex = await this.loadSparseIndex(field)
|
|
||||||
if (!sparseIndex) {
|
|
||||||
// Create new sparse index
|
|
||||||
const stats = this.fieldStats.get(field)
|
|
||||||
const chunkSize = stats
|
|
||||||
? this.chunkingStrategy.getOptimalChunkSize({
|
|
||||||
uniqueValues: stats.cardinality.uniqueValues,
|
|
||||||
distribution: stats.cardinality.distribution,
|
|
||||||
avgIdsPerValue: stats.cardinality.totalValues / Math.max(1, stats.cardinality.uniqueValues)
|
|
||||||
})
|
|
||||||
: 50
|
|
||||||
|
|
||||||
sparseIndex = new SparseIndex(field, chunkSize)
|
// removeFromChunkedIndex — DELETED. Column store handles removes via global deleted bitmap.
|
||||||
}
|
|
||||||
|
|
||||||
const normalizedValue = this.normalizeValue(value, field)
|
|
||||||
|
|
||||||
// Find existing chunk for this value (check zone maps)
|
|
||||||
const candidateChunkIds = sparseIndex.findChunksForValue(normalizedValue)
|
|
||||||
|
|
||||||
let targetChunk: ChunkData | null = null
|
|
||||||
let targetChunkId: number | null = null
|
|
||||||
|
|
||||||
// Try to find an existing chunk with this value
|
|
||||||
for (const chunkId of candidateChunkIds) {
|
|
||||||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
|
||||||
if (chunk && chunk.entries.has(normalizedValue)) {
|
|
||||||
targetChunk = chunk
|
|
||||||
targetChunkId = chunkId
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no chunk has this value, find chunk with space or create new one
|
|
||||||
if (!targetChunk) {
|
|
||||||
// Find a chunk with available space
|
|
||||||
for (const chunkId of sparseIndex.getAllChunkIds()) {
|
|
||||||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
|
||||||
const descriptor = sparseIndex.getChunk(chunkId)
|
|
||||||
if (chunk && descriptor && chunk.entries.size < descriptor.splitThreshold) {
|
|
||||||
targetChunk = chunk
|
|
||||||
targetChunkId = chunkId
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create new chunk if needed
|
|
||||||
if (!targetChunk) {
|
|
||||||
targetChunk = await this.chunkManager.createChunk(field)
|
|
||||||
targetChunkId = targetChunk.chunkId
|
|
||||||
|
|
||||||
// Register in sparse index
|
|
||||||
const descriptor: ChunkDescriptor = {
|
|
||||||
chunkId: targetChunk.chunkId,
|
|
||||||
field,
|
|
||||||
valueCount: 0,
|
|
||||||
idCount: 0,
|
|
||||||
zoneMap: { min: null, max: null, count: 0, hasNulls: false },
|
|
||||||
lastUpdated: Date.now(),
|
|
||||||
splitThreshold: 80,
|
|
||||||
mergeThreshold: 20
|
|
||||||
}
|
|
||||||
sparseIndex.registerChunk(descriptor)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to chunk
|
|
||||||
await this.chunkManager.addToChunk(targetChunk, normalizedValue, id)
|
|
||||||
// Defer chunk save — mark dirty instead of writing immediately
|
|
||||||
this.dirtyChunks.set(`${targetChunk.field}:${targetChunk.chunkId}`, targetChunk)
|
|
||||||
|
|
||||||
// Update chunk descriptor in sparse index
|
|
||||||
const updatedZoneMap = this.chunkManager.calculateZoneMap(targetChunk)
|
|
||||||
const updatedBloomFilter = this.chunkManager.createBloomFilter(targetChunk)
|
|
||||||
|
|
||||||
sparseIndex.updateChunk(targetChunkId!, {
|
|
||||||
valueCount: targetChunk.entries.size,
|
|
||||||
idCount: Array.from(targetChunk.entries.values()).reduce((sum, bitmap) => sum + bitmap.size, 0),
|
|
||||||
zoneMap: updatedZoneMap,
|
|
||||||
lastUpdated: Date.now()
|
|
||||||
})
|
|
||||||
|
|
||||||
// Update bloom filter
|
|
||||||
const descriptor = sparseIndex.getChunk(targetChunkId!)
|
|
||||||
if (descriptor) {
|
|
||||||
sparseIndex.registerChunk(descriptor, updatedBloomFilter)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if chunk needs splitting
|
|
||||||
if (targetChunk.entries.size > 80) {
|
|
||||||
const { chunk1, chunk2 } = await this.splitChunkDeferred(targetChunk, sparseIndex)
|
|
||||||
// Mark split result chunks as dirty instead of the original
|
|
||||||
this.dirtyChunks.delete(`${targetChunk.field}:${targetChunk.chunkId}`)
|
|
||||||
this.dirtyChunks.set(`${chunk1.field}:${chunk1.chunkId}`, chunk1)
|
|
||||||
this.dirtyChunks.set(`${chunk2.field}:${chunk2.chunkId}`, chunk2)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Defer sparse index save — mark dirty instead of writing immediately
|
|
||||||
this.dirtySparseIndices.set(field, sparseIndex)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Remove ID from chunked index
|
|
||||||
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
|
|
||||||
*/
|
|
||||||
private async removeFromChunkedIndex(field: string, value: any, id: string): Promise<void> {
|
|
||||||
// Load sparse index via UnifiedCache (lazy loading)
|
|
||||||
const sparseIndex = await this.loadSparseIndex(field)
|
|
||||||
if (!sparseIndex) {
|
|
||||||
return // No chunked index exists
|
|
||||||
}
|
|
||||||
|
|
||||||
const normalizedValue = this.normalizeValue(value, field)
|
|
||||||
const candidateChunkIds = sparseIndex.findChunksForValue(normalizedValue)
|
|
||||||
|
|
||||||
for (const chunkId of candidateChunkIds) {
|
|
||||||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
|
||||||
if (chunk && chunk.entries.has(normalizedValue)) {
|
|
||||||
await this.chunkManager.removeFromChunk(chunk, normalizedValue, id)
|
|
||||||
// Defer chunk save — mark dirty instead of writing immediately
|
|
||||||
this.dirtyChunks.set(`${chunk.field}:${chunk.chunkId}`, chunk)
|
|
||||||
|
|
||||||
// Update sparse index
|
|
||||||
const updatedZoneMap = this.chunkManager.calculateZoneMap(chunk)
|
|
||||||
sparseIndex.updateChunk(chunkId, {
|
|
||||||
valueCount: chunk.entries.size,
|
|
||||||
idCount: Array.from(chunk.entries.values()).reduce((sum, bitmap) => sum + bitmap.size, 0),
|
|
||||||
zoneMap: updatedZoneMap,
|
|
||||||
lastUpdated: Date.now()
|
|
||||||
})
|
|
||||||
|
|
||||||
// Defer sparse index save — mark dirty instead of writing immediately
|
|
||||||
this.dirtySparseIndices.set(field, sparseIndex)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get IDs matching a range query using zone maps
|
* Get IDs matching a range query using zone maps
|
||||||
|
|
@ -1238,25 +1017,7 @@ export class MetadataIndexManager {
|
||||||
return `field_${field}`
|
return `field_${field}`
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
// getValueChunkFilename, makeSafeFilename — DELETED. Sparse index file naming no longer needed.
|
||||||
* Generate value chunk filename for scalable storage
|
|
||||||
*/
|
|
||||||
private getValueChunkFilename(field: string, value: any, chunkIndex: number = 0): string {
|
|
||||||
const normalizedValue = this.normalizeValue(value, field) // Pass field for bucketing!
|
|
||||||
const safeValue = this.makeSafeFilename(normalizedValue)
|
|
||||||
return `${field}_${safeValue}_chunk${chunkIndex}`
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Make a value safe for use in filenames
|
|
||||||
*/
|
|
||||||
private makeSafeFilename(value: string): string {
|
|
||||||
// Replace unsafe characters and limit length
|
|
||||||
return value
|
|
||||||
.replace(/[^a-zA-Z0-9-_]/g, '_')
|
|
||||||
.substring(0, 50)
|
|
||||||
.toLowerCase()
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Normalize value for consistent indexing with VALUE-BASED temporal detection
|
* Normalize value for consistent indexing with VALUE-BASED temporal detection
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue