David Snelling
de79d6b5a4
Each file opened one or more Brainy instances (beforeEach, or a small per-test helper like migration-gate-family-scoped's module-level seed()) and never closed them. migration-gate-family-scoped.test.ts now tracks every brain seed() hands back in a describe-scoped array drained by afterEach, since the helper itself lives outside the describe block.
262 lines
11 KiB
TypeScript
262 lines
11 KiB
TypeScript
/**
|
|
* @module tests/unit/utils/metadataIndex-sparse-range-collation
|
|
* @description RANGE QUERIES ON THE LEGACY SPARSE INDEX — order, or a refusal.
|
|
* Never a confidently ordered wrong answer.
|
|
*
|
|
* THE TWO RANGE PATHS. `getIdsForRange` routes a `gte` / `lt` / `between` two
|
|
* ways. The column store compares RAW values and is correct. The legacy sparse
|
|
* chunk index — the pre-7.20.0 fallback, still read for workspaces that have
|
|
* not been rebuilt — compared `normalizeValue()` output, and `normalizeValue`
|
|
* carries an escape hatch that destroys order on purpose: a string over 100
|
|
* characters is replaced by a short hash so it can serve as a filesystem-safe
|
|
* key. Ordering hashes ranks rows by digest.
|
|
*
|
|
* THE DEFECT, IN TWO SHAPES.
|
|
*
|
|
* (a) A LONG BOUND against ordinary values. `where: { title: { gte: <a
|
|
* 120-character string> } }` collapsed the BOUND to `__HASH_…`, whose
|
|
* leading underscores sort below every letter — so a bound that should
|
|
* have excluded everything matched the entire field instead. This is the
|
|
* shape that reaches a caller who never stored a long value at all.
|
|
*
|
|
* (b) LONG VALUES in the index. A field whose values ran long was persisted
|
|
* hashed, so its order is not recoverable from this index at all. The old
|
|
* code compared the digests anyway and returned a subset chosen by hash.
|
|
*
|
|
* THE LAW. Bounds are normalized WITHOUT the hash escape hatch, so a long
|
|
* bound stays comparable — (a) is simply fixed. Where the persisted KEY is a
|
|
* hash, the order does not exist to be computed, and the query throws a typed
|
|
* `BrainyError('INVALID_QUERY')` naming the field and the cure — (b) is
|
|
* refused by name. Loud beats wrong.
|
|
*
|
|
* THE FIXTURE is a genuine legacy index: it is written through the same
|
|
* `ChunkManager` / `SparseIndex` doors a pre-7.20.0 engine wrote through, with
|
|
* keys normalized exactly as that engine normalized them, into a field the
|
|
* column store does not serve. The chunk WRITE path was removed in 11be039, so
|
|
* this is the only way the shape the read path exists for can be built.
|
|
*
|
|
* NOT CLAIMED HERE. The persisted keys are also lower-cased and trimmed by
|
|
* `normalizeValue`, so this path's string ranges are case-INSENSITIVE where
|
|
* the column store's are not. The raw values are not in the index to compare —
|
|
* that divergence is a property of the bytes on disk and it ends when the
|
|
* column store adopts the field. It is named in `getIdsFromChunksForRange`'s
|
|
* doc comment rather than papered over.
|
|
*/
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
import { Brainy } from '../../../src/brainy'
|
|
import { NounType } from '../../../src/types/graphTypes'
|
|
import { SparseIndex, ChunkManager } from '../../../src/utils/metadataIndexChunking'
|
|
import { BrainyError } from '../../../src/errors/brainyError'
|
|
|
|
/** The field the legacy index covers — deliberately never given to a row, so
|
|
* the column store never learns it and the sparse fallback is the only path. */
|
|
const FIELD = 'legacyTitle'
|
|
|
|
/**
|
|
* Write a legacy sparse index for `field` exactly as a pre-7.20.0 engine did:
|
|
* one chunk, keys normalized through the index's own `normalizeValue`, ids as
|
|
* roaring bitmaps, a zone map and a bloom filter over the chunk.
|
|
*
|
|
* @param brain - The live brain whose metadata index gains the legacy field.
|
|
* @param field - Field name to index.
|
|
* @param valueToIds - Raw value → the entity ids that carried it.
|
|
*/
|
|
async function writeLegacySparseIndex(
|
|
brain: any,
|
|
field: string,
|
|
valueToIds: Array<[string, string[]]>
|
|
): Promise<void> {
|
|
const index = brain.metadataIndex
|
|
const chunkManager: ChunkManager = index.chunkManager
|
|
const sparseIndex = new SparseIndex(field)
|
|
|
|
// The keys a pre-7.20.0 writer persisted: normalizeValue output, hash escape
|
|
// hatch and all. This is what makes the fixture the real shape.
|
|
const chunk = await chunkManager.createChunk(field)
|
|
for (const [value, ids] of valueToIds) {
|
|
const key = index.normalizeValue(value, field)
|
|
for (const id of ids) await chunkManager.addToChunk(chunk, key, id)
|
|
}
|
|
await chunkManager.saveChunk(chunk)
|
|
|
|
sparseIndex.registerChunk(
|
|
{
|
|
chunkId: chunk.chunkId,
|
|
field,
|
|
valueCount: chunk.entries.size,
|
|
idCount: Array.from(chunk.entries.values()).reduce((s: number, b: any) => s + b.size, 0),
|
|
zoneMap: (chunkManager as any).calculateZoneMap(chunk),
|
|
lastUpdated: Date.now(),
|
|
splitThreshold: 80,
|
|
mergeThreshold: 20
|
|
},
|
|
chunkManager.createBloomFilter(chunk)
|
|
)
|
|
|
|
await index.saveSparseIndex(field, sparseIndex)
|
|
}
|
|
|
|
/** A deterministic string of `n` characters starting with `lead`. */
|
|
function longString(lead: string, n: number): string {
|
|
return lead + 'x'.repeat(n - lead.length)
|
|
}
|
|
|
|
describe('legacy sparse index: range queries order values, or refuse', () => {
|
|
let brain: Brainy<any>
|
|
let index: any
|
|
let ids: string[]
|
|
|
|
beforeEach(async () => {
|
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
await brain.init()
|
|
index = (brain as any).metadataIndex
|
|
|
|
// Rows exist (so the id mapper can resolve them) but carry NO `legacyTitle`
|
|
// — the column store must not serve the field the pins query.
|
|
ids = []
|
|
for (let i = 0; i < 3; i++) {
|
|
const id = `row-${i}`
|
|
await brain.add({ id, data: `row ${i}`, type: NounType.Thing, metadata: { lane: 'a' }, vector: [] })
|
|
ids.push(id)
|
|
}
|
|
expect(index.columnStore.hasField(FIELD)).toBe(false)
|
|
})
|
|
|
|
afterEach(async () => {
|
|
await brain.close()
|
|
})
|
|
|
|
describe('(a) a long BOUND against ordinary short values', () => {
|
|
// 'apple' < 'mango' < 'zebra', and every bound below is compared against
|
|
// these three raw keys.
|
|
beforeEach(async () => {
|
|
await writeLegacySparseIndex(brain, FIELD, [
|
|
['apple', [ids[0]]],
|
|
['mango', [ids[1]]],
|
|
['zebra', [ids[2]]]
|
|
])
|
|
})
|
|
|
|
it('the fixture: the values are stored raw, the long bound is what hashes', () => {
|
|
expect(index.normalizeValue('apple', FIELD)).toBe('apple')
|
|
// The bound is what the old code collapsed — and a digest sorts below
|
|
// every letter, which is exactly why `gte` matched everything.
|
|
const bound = longString('zzz', 120)
|
|
expect(index.normalizeValue(bound, FIELD)).toMatch(/^__HASH_/)
|
|
expect(index.normalizeValue(bound, FIELD) < 'apple').toBe(true)
|
|
})
|
|
|
|
it('gte a bound above every value matches NOTHING (it used to match all)', async () => {
|
|
const bound = longString('zzz', 120)
|
|
const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
|
|
expect(matched).toEqual([])
|
|
})
|
|
|
|
it('lte a bound above every value matches EVERY value', async () => {
|
|
const bound = longString('zzz', 120)
|
|
const matched = await index.getIdsForRange(FIELD, undefined, bound, true, true)
|
|
expect(matched).toHaveLength(3)
|
|
})
|
|
|
|
it('gte a long bound below every value matches every value', async () => {
|
|
const bound = longString('aaa', 120)
|
|
const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
|
|
expect(matched).toHaveLength(3)
|
|
})
|
|
|
|
it('a long bound orders BETWEEN the values, not below all of them', async () => {
|
|
// 'mmm…' sits between 'mango' and 'zebra'.
|
|
const bound = longString('mmm', 120)
|
|
const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true)
|
|
expect(matched).toHaveLength(1)
|
|
})
|
|
|
|
it('short bounds are unchanged — the ordinary case still orders correctly', async () => {
|
|
expect(await index.getIdsForRange(FIELD, 'b', undefined, true, true)).toHaveLength(2)
|
|
expect(await index.getIdsForRange(FIELD, undefined, 'n', true, true)).toHaveLength(2)
|
|
expect(await index.getIdsForRange(FIELD, 'b', 'n', true, true)).toHaveLength(1)
|
|
// Strict bounds stay strict.
|
|
expect(await index.getIdsForRange(FIELD, 'mango', undefined, false, true)).toHaveLength(1)
|
|
expect(await index.getIdsForRange(FIELD, 'mango', undefined, true, true)).toHaveLength(2)
|
|
})
|
|
})
|
|
|
|
describe('(b) long VALUES — the index holds hashes, so the range is refused', () => {
|
|
beforeEach(async () => {
|
|
await writeLegacySparseIndex(brain, FIELD, [
|
|
[longString('alpha', 140), [ids[0]]],
|
|
[longString('mike', 140), [ids[1]]],
|
|
[longString('zulu', 140), [ids[2]]]
|
|
])
|
|
})
|
|
|
|
it('the fixture: the persisted keys really are hashes', async () => {
|
|
const chunk = await index.chunkManager.loadChunk(FIELD, 0)
|
|
const keys = Array.from(chunk.entries.keys()) as string[]
|
|
expect(keys).toHaveLength(3)
|
|
for (const k of keys) expect(k).toMatch(/^__HASH_/)
|
|
// And their digest order is NOT their value order — the wrong answer the
|
|
// old code returned was wrong, not merely arbitrary.
|
|
const digestOrder = [...keys].sort()
|
|
const valueOrder = [
|
|
index.normalizeValue(longString('alpha', 140), FIELD),
|
|
index.normalizeValue(longString('mike', 140), FIELD),
|
|
index.normalizeValue(longString('zulu', 140), FIELD)
|
|
]
|
|
expect(digestOrder).not.toEqual(valueOrder)
|
|
})
|
|
|
|
it('a range over the hashed field throws a typed refusal naming the field', async () => {
|
|
await expect(
|
|
index.getIdsForRange(FIELD, longString('mike', 140), undefined, true, true)
|
|
).rejects.toThrow(BrainyError)
|
|
|
|
const err = await index
|
|
.getIdsForRange(FIELD, longString('mike', 140), undefined, true, true)
|
|
.catch((e: any) => e)
|
|
expect(err).toBeInstanceOf(BrainyError)
|
|
expect(err.type).toBe('INVALID_QUERY')
|
|
expect(err.message).toContain(FIELD)
|
|
expect(err.message).toContain('hash')
|
|
// The cure is named, not left to the caller to guess.
|
|
expect(err.message).toContain('repairIndex')
|
|
})
|
|
|
|
it('every range shape refuses — gte, lte and between alike', async () => {
|
|
const lo = longString('alpha', 140)
|
|
const hi = longString('zulu', 140)
|
|
for (const [min, max] of [
|
|
[lo, undefined],
|
|
[undefined, hi],
|
|
[lo, hi]
|
|
] as Array<[any, any]>) {
|
|
const err = await index.getIdsForRange(FIELD, min, max, true, true).catch((e: any) => e)
|
|
expect(err).toBeInstanceOf(BrainyError)
|
|
expect(err.type).toBe('INVALID_QUERY')
|
|
}
|
|
})
|
|
|
|
it('EQUALITY still works on the hashed field — only ordering is refused', async () => {
|
|
const matched = await index.getIds(FIELD, longString('mike', 140))
|
|
expect(matched).toHaveLength(1)
|
|
})
|
|
})
|
|
|
|
describe('numeric ranges on the legacy path are untouched', () => {
|
|
beforeEach(async () => {
|
|
await writeLegacySparseIndex(brain, FIELD, [
|
|
['5', [ids[0]]],
|
|
['50', [ids[1]]],
|
|
['500', [ids[2]]]
|
|
])
|
|
})
|
|
|
|
it('numbers still compare numerically, not lexicographically', async () => {
|
|
// The whole point of compareNormalizedValues: "50" < "500" numerically
|
|
// even though "500" < "50" would hold as strings by prefix.
|
|
expect(await index.getIdsForRange(FIELD, 10, undefined, true, true)).toHaveLength(2)
|
|
expect(await index.getIdsForRange(FIELD, undefined, 100, true, true)).toHaveLength(2)
|
|
expect(await index.getIdsForRange(FIELD, 10, 100, true, true)).toHaveLength(1)
|
|
})
|
|
})
|
|
})
|