refactor: delete dead sparse index write path
Column store handles all writes and primary queries. Sparse index write methods are unreachable: Deleted: addToChunkedIndex (~103 lines), removeFromChunkedIndex (~37 lines), splitChunkDeferred (~82 lines), getValueChunkFilename + makeSafeFilename (~20 lines). Total: ~240 lines of dead write-path code. Sparse index read methods kept as migration fallback for existing workspaces. Will be deleted once lazy migration (buildFromStorage) ships.
This commit is contained in:
parent
46583f2d9b
commit
11be039ae8
1 changed files with 4 additions and 243 deletions
|
|
@ -757,88 +757,7 @@ export class MetadataIndexManager {
|
|||
this.dirtySparseIndices.clear()
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a chunk without saving immediately — returns the new chunks for deferred save.
|
||||
* Used by addToChunkedIndex() to keep splits within the deferred write batch.
|
||||
*/
|
||||
private async splitChunkDeferred(
|
||||
chunk: ChunkData,
|
||||
sparseIndex: SparseIndex
|
||||
): Promise<{ chunk1: ChunkData; chunk2: ChunkData }> {
|
||||
const values = Array.from(chunk.entries.keys()).sort()
|
||||
const midpoint = Math.floor(values.length / 2)
|
||||
|
||||
// Create two new chunks with roaring bitmaps
|
||||
const entries1 = new Map<string, RoaringBitmap32>()
|
||||
const entries2 = new Map<string, RoaringBitmap32>()
|
||||
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
const value = values[i]
|
||||
const bitmap = chunk.entries.get(value)!
|
||||
|
||||
if (i < midpoint) {
|
||||
entries1.set(value, new RoaringBitmap32(bitmap.toArray()))
|
||||
} else {
|
||||
entries2.set(value, new RoaringBitmap32(bitmap.toArray()))
|
||||
}
|
||||
}
|
||||
|
||||
// Create chunk objects without saving (just allocate IDs and set up data)
|
||||
const chunkId1 = this.chunkManager['getNextChunkId'](chunk.field)
|
||||
const chunk1: ChunkData = {
|
||||
chunkId: chunkId1,
|
||||
field: chunk.field,
|
||||
entries: entries1,
|
||||
lastUpdated: Date.now()
|
||||
}
|
||||
|
||||
const chunkId2 = this.chunkManager['getNextChunkId'](chunk.field)
|
||||
const chunk2: ChunkData = {
|
||||
chunkId: chunkId2,
|
||||
field: chunk.field,
|
||||
entries: entries2,
|
||||
lastUpdated: Date.now()
|
||||
}
|
||||
|
||||
// Update chunk cache (for read-after-write consistency within this operation)
|
||||
this.chunkManager['chunkCache'].set(`${chunk.field}:${chunkId1}`, chunk1)
|
||||
this.chunkManager['chunkCache'].set(`${chunk.field}:${chunkId2}`, chunk2)
|
||||
|
||||
// Update sparse index
|
||||
sparseIndex.removeChunk(chunk.chunkId)
|
||||
|
||||
const descriptor1: ChunkDescriptor = {
|
||||
chunkId: chunk1.chunkId,
|
||||
field: chunk1.field,
|
||||
valueCount: entries1.size,
|
||||
idCount: Array.from(entries1.values()).reduce((sum, bitmap) => sum + bitmap.size, 0),
|
||||
zoneMap: this.chunkManager.calculateZoneMap(chunk1),
|
||||
lastUpdated: Date.now(),
|
||||
splitThreshold: 80,
|
||||
mergeThreshold: 20
|
||||
}
|
||||
|
||||
const descriptor2: ChunkDescriptor = {
|
||||
chunkId: chunk2.chunkId,
|
||||
field: chunk2.field,
|
||||
valueCount: entries2.size,
|
||||
idCount: Array.from(entries2.values()).reduce((sum, bitmap) => sum + bitmap.size, 0),
|
||||
zoneMap: this.chunkManager.calculateZoneMap(chunk2),
|
||||
lastUpdated: Date.now(),
|
||||
splitThreshold: 80,
|
||||
mergeThreshold: 20
|
||||
}
|
||||
|
||||
sparseIndex.registerChunk(descriptor1, this.chunkManager.createBloomFilter(chunk1))
|
||||
sparseIndex.registerChunk(descriptor2, this.chunkManager.createBloomFilter(chunk2))
|
||||
|
||||
// Delete old chunk from storage (this still writes immediately as it's a deletion)
|
||||
await this.chunkManager.deleteChunk(chunk.field, chunk.chunkId)
|
||||
|
||||
prodLog.debug(`Split chunk ${chunk.field}:${chunk.chunkId} into ${chunk1.chunkId} and ${chunk2.chunkId} (deferred save)`)
|
||||
|
||||
return { chunk1, chunk2 }
|
||||
}
|
||||
// splitChunkDeferred — DELETED. Only called from addToChunkedIndex (also deleted).
|
||||
|
||||
/**
|
||||
* Get IDs for a value using chunked sparse index with roaring bitmaps
|
||||
|
|
@ -1047,149 +966,9 @@ export class MetadataIndexManager {
|
|||
return result.size > 0 ? this.idMapper.intsIterableToUuids(result) : []
|
||||
}
|
||||
|
||||
/**
|
||||
* Add value-ID mapping to chunked index
|
||||
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
|
||||
*/
|
||||
private async addToChunkedIndex(field: string, value: any, id: string): Promise<void> {
|
||||
// Load or create sparse index via UnifiedCache (lazy loading)
|
||||
let sparseIndex = await this.loadSparseIndex(field)
|
||||
if (!sparseIndex) {
|
||||
// Create new sparse index
|
||||
const stats = this.fieldStats.get(field)
|
||||
const chunkSize = stats
|
||||
? this.chunkingStrategy.getOptimalChunkSize({
|
||||
uniqueValues: stats.cardinality.uniqueValues,
|
||||
distribution: stats.cardinality.distribution,
|
||||
avgIdsPerValue: stats.cardinality.totalValues / Math.max(1, stats.cardinality.uniqueValues)
|
||||
})
|
||||
: 50
|
||||
// addToChunkedIndex — DELETED. Column store handles all writes.
|
||||
|
||||
sparseIndex = new SparseIndex(field, chunkSize)
|
||||
}
|
||||
|
||||
const normalizedValue = this.normalizeValue(value, field)
|
||||
|
||||
// Find existing chunk for this value (check zone maps)
|
||||
const candidateChunkIds = sparseIndex.findChunksForValue(normalizedValue)
|
||||
|
||||
let targetChunk: ChunkData | null = null
|
||||
let targetChunkId: number | null = null
|
||||
|
||||
// Try to find an existing chunk with this value
|
||||
for (const chunkId of candidateChunkIds) {
|
||||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
||||
if (chunk && chunk.entries.has(normalizedValue)) {
|
||||
targetChunk = chunk
|
||||
targetChunkId = chunkId
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If no chunk has this value, find chunk with space or create new one
|
||||
if (!targetChunk) {
|
||||
// Find a chunk with available space
|
||||
for (const chunkId of sparseIndex.getAllChunkIds()) {
|
||||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
||||
const descriptor = sparseIndex.getChunk(chunkId)
|
||||
if (chunk && descriptor && chunk.entries.size < descriptor.splitThreshold) {
|
||||
targetChunk = chunk
|
||||
targetChunkId = chunkId
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create new chunk if needed
|
||||
if (!targetChunk) {
|
||||
targetChunk = await this.chunkManager.createChunk(field)
|
||||
targetChunkId = targetChunk.chunkId
|
||||
|
||||
// Register in sparse index
|
||||
const descriptor: ChunkDescriptor = {
|
||||
chunkId: targetChunk.chunkId,
|
||||
field,
|
||||
valueCount: 0,
|
||||
idCount: 0,
|
||||
zoneMap: { min: null, max: null, count: 0, hasNulls: false },
|
||||
lastUpdated: Date.now(),
|
||||
splitThreshold: 80,
|
||||
mergeThreshold: 20
|
||||
}
|
||||
sparseIndex.registerChunk(descriptor)
|
||||
}
|
||||
|
||||
// Add to chunk
|
||||
await this.chunkManager.addToChunk(targetChunk, normalizedValue, id)
|
||||
// Defer chunk save — mark dirty instead of writing immediately
|
||||
this.dirtyChunks.set(`${targetChunk.field}:${targetChunk.chunkId}`, targetChunk)
|
||||
|
||||
// Update chunk descriptor in sparse index
|
||||
const updatedZoneMap = this.chunkManager.calculateZoneMap(targetChunk)
|
||||
const updatedBloomFilter = this.chunkManager.createBloomFilter(targetChunk)
|
||||
|
||||
sparseIndex.updateChunk(targetChunkId!, {
|
||||
valueCount: targetChunk.entries.size,
|
||||
idCount: Array.from(targetChunk.entries.values()).reduce((sum, bitmap) => sum + bitmap.size, 0),
|
||||
zoneMap: updatedZoneMap,
|
||||
lastUpdated: Date.now()
|
||||
})
|
||||
|
||||
// Update bloom filter
|
||||
const descriptor = sparseIndex.getChunk(targetChunkId!)
|
||||
if (descriptor) {
|
||||
sparseIndex.registerChunk(descriptor, updatedBloomFilter)
|
||||
}
|
||||
|
||||
// Check if chunk needs splitting
|
||||
if (targetChunk.entries.size > 80) {
|
||||
const { chunk1, chunk2 } = await this.splitChunkDeferred(targetChunk, sparseIndex)
|
||||
// Mark split result chunks as dirty instead of the original
|
||||
this.dirtyChunks.delete(`${targetChunk.field}:${targetChunk.chunkId}`)
|
||||
this.dirtyChunks.set(`${chunk1.field}:${chunk1.chunkId}`, chunk1)
|
||||
this.dirtyChunks.set(`${chunk2.field}:${chunk2.chunkId}`, chunk2)
|
||||
}
|
||||
|
||||
// Defer sparse index save — mark dirty instead of writing immediately
|
||||
this.dirtySparseIndices.set(field, sparseIndex)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove ID from chunked index
|
||||
* Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map)
|
||||
*/
|
||||
private async removeFromChunkedIndex(field: string, value: any, id: string): Promise<void> {
|
||||
// Load sparse index via UnifiedCache (lazy loading)
|
||||
const sparseIndex = await this.loadSparseIndex(field)
|
||||
if (!sparseIndex) {
|
||||
return // No chunked index exists
|
||||
}
|
||||
|
||||
const normalizedValue = this.normalizeValue(value, field)
|
||||
const candidateChunkIds = sparseIndex.findChunksForValue(normalizedValue)
|
||||
|
||||
for (const chunkId of candidateChunkIds) {
|
||||
const chunk = await this.chunkManager.loadChunk(field, chunkId)
|
||||
if (chunk && chunk.entries.has(normalizedValue)) {
|
||||
await this.chunkManager.removeFromChunk(chunk, normalizedValue, id)
|
||||
// Defer chunk save — mark dirty instead of writing immediately
|
||||
this.dirtyChunks.set(`${chunk.field}:${chunk.chunkId}`, chunk)
|
||||
|
||||
// Update sparse index
|
||||
const updatedZoneMap = this.chunkManager.calculateZoneMap(chunk)
|
||||
sparseIndex.updateChunk(chunkId, {
|
||||
valueCount: chunk.entries.size,
|
||||
idCount: Array.from(chunk.entries.values()).reduce((sum, bitmap) => sum + bitmap.size, 0),
|
||||
zoneMap: updatedZoneMap,
|
||||
lastUpdated: Date.now()
|
||||
})
|
||||
|
||||
// Defer sparse index save — mark dirty instead of writing immediately
|
||||
this.dirtySparseIndices.set(field, sparseIndex)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// removeFromChunkedIndex — DELETED. Column store handles removes via global deleted bitmap.
|
||||
|
||||
/**
|
||||
* Get IDs matching a range query using zone maps
|
||||
|
|
@ -1238,25 +1017,7 @@ export class MetadataIndexManager {
|
|||
return `field_${field}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate value chunk filename for scalable storage
|
||||
*/
|
||||
private getValueChunkFilename(field: string, value: any, chunkIndex: number = 0): string {
|
||||
const normalizedValue = this.normalizeValue(value, field) // Pass field for bucketing!
|
||||
const safeValue = this.makeSafeFilename(normalizedValue)
|
||||
return `${field}_${safeValue}_chunk${chunkIndex}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a value safe for use in filenames
|
||||
*/
|
||||
private makeSafeFilename(value: string): string {
|
||||
// Replace unsafe characters and limit length
|
||||
return value
|
||||
.replace(/[^a-zA-Z0-9-_]/g, '_')
|
||||
.substring(0, 50)
|
||||
.toLowerCase()
|
||||
}
|
||||
// getValueChunkFilename, makeSafeFilename — DELETED. Sparse index file naming no longer needed.
|
||||
|
||||
/**
|
||||
* Normalize value for consistent indexing with VALUE-BASED temporal detection
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue