feat(8.0): u64 BigInt graph provider contract — punch list a-d,g,h
GraphIndexProvider now speaks BigInt at the boundary (D.2 mirror): - getNeighbors/getVerbIdsBySource/getVerbIdsByTarget take entity ints and return entity/verb ints as bigint[] - new REQUIRED verbIntsToIds(bigint[]) batch reverse resolver (L.7 identity-fingerprint design — verb ids are UUIDs by contract, so the provider-side interning is losslessly reversible) - addVerb(verb, sourceInt, targetInt) returns the interned verb int; removeVerb(verbId) joins the contract Coordinator (brainy.ts) owns ALL UUID <-> int conversion: getOrAssign on writes, getInt on reads (unmapped UUID -> empty result without calling the provider), getUuid / verbIntsToIds on returns, plus a bounded ~100k-entry insertion-order warm cache for verb-int -> verb-id pairs fed by addVerb returns and resolver results. GraphVerb gains derived sourceInt/targetInt (populated at add time, never persisted). findConnectedSubtype gains a native fast path that routes single-type single-subtype outgoing BFS through the provider when available. JS GraphAdjacencyIndex satisfies the contract while staying string/u32-keyed internally: entity ints resolve through the shared entity-id mapper (threaded in by the coordinator on init/fork/checkout), verb ints come from an in-process append-only interning map re-derived from storage on rebuild/cold-start. ColumnStoreProvider widens the same way: addEntity/ removeEntity take bigint, sortTopK/filteredSortTopK return bigint[]. relate() now rejects a caller-supplied id with a teaching error — verb ids are brainy-generated UUIDs by contract in 8.0 (previously a passed id was silently ignored). No Roaring64 provider-boundary decode site exists yet; the JS-internal column store stays Roaring32 and the Treemap decoder lands with the first consumer of provider-returned filter buffers. Public brain API unchanged. 1413 tests green (+10 new BigInt contract tests).
This commit is contained in:
parent
62f6472fa0
commit
2427bb7960
17 changed files with 1164 additions and 328 deletions
|
|
@ -44,7 +44,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
|
||||
it('flushBuffer writes segment bytes as a raw blob at the cortex-shared key', async () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i * 10 })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
|
|
@ -63,9 +63,9 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
|
||||
it('flushBuffer writes the DELETED bitmap as a raw blob too (not an envelope)', async () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i })
|
||||
}
|
||||
store.removeEntity(idMapper.getInt('u-1')!)
|
||||
store.removeEntity(BigInt(idMapper.getInt('u-1')!))
|
||||
await store.flush()
|
||||
|
||||
const rawDel = await (storage as any).loadBinaryBlob('_column_index/score/DELETED')
|
||||
|
|
@ -135,7 +135,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
const readStore = new ColumnStore()
|
||||
await readStore.init(storage, idMapper)
|
||||
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i)))
|
||||
expect(sortedUuids).toEqual(['legacy-a', 'legacy-b', 'legacy-c'])
|
||||
await readStore.close()
|
||||
})
|
||||
|
|
@ -153,15 +153,15 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
|
||||
// Add a couple of live entities so a manifest exists (init only loads
|
||||
// DELETED for fields with a manifest on disk).
|
||||
store.addEntity(idMapper.getOrAssign('alive-1'), { score: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('alive-2'), { score: 200 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('alive-1')), { score: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('alive-2')), { score: 200 })
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
||||
const readStore = new ColumnStore()
|
||||
await readStore.init(storage, idMapper)
|
||||
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i)))
|
||||
|
||||
// The dead entity must NOT appear; the alive ones must.
|
||||
expect(sortedUuids).not.toContain('dead')
|
||||
|
|
@ -176,7 +176,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
|
||||
it('write (new format) → re-init → read returns the same data', async () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i * 10 })
|
||||
}
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
|
@ -184,7 +184,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
const readStore = new ColumnStore()
|
||||
await readStore.init(storage, idMapper)
|
||||
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i)))
|
||||
expect(sortedUuids).toEqual(['u-0', 'u-1', 'u-2', 'u-3', 'u-4'])
|
||||
await readStore.close()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -35,48 +35,48 @@ describe('ColumnStore', () => {
|
|||
|
||||
describe('sortTopK', () => {
|
||||
it('returns entities sorted by numeric field ascending', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['b', 'c', 'a'])
|
||||
})
|
||||
|
||||
it('returns entities sorted by numeric field descending', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['a', 'c', 'b'])
|
||||
})
|
||||
|
||||
it('respects limit K', async () => {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`e${i}`), { createdAt: i * 1000 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { createdAt: i * 1000 })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'desc', 5)
|
||||
expect(result).toHaveLength(5)
|
||||
// Should be the 5 highest timestamps
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['e49', 'e48', 'e47', 'e46', 'e45'])
|
||||
})
|
||||
|
||||
it('sorts string fields lexicographically', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'pending' })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { status: 'inactive' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'pending' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { status: 'inactive' })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('status', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['b', 'c', 'a']) // active, inactive, pending
|
||||
})
|
||||
|
||||
|
|
@ -97,16 +97,16 @@ describe('ColumnStore', () => {
|
|||
const idC = idMapper.getOrAssign('c')
|
||||
const idD = idMapper.getOrAssign('d')
|
||||
|
||||
store.addEntity(idA, { createdAt: 400 })
|
||||
store.addEntity(idB, { createdAt: 100 })
|
||||
store.addEntity(idC, { createdAt: 300 })
|
||||
store.addEntity(idD, { createdAt: 200 })
|
||||
store.addEntity(BigInt(idA), { createdAt: 400 })
|
||||
store.addEntity(BigInt(idB), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idC), { createdAt: 300 })
|
||||
store.addEntity(BigInt(idD), { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
// Filter: only B and C
|
||||
const bitmap = new RoaringBitmap32([idB, idC])
|
||||
const result = await store.filteredSortTopK(bitmap, 'createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['c', 'b']) // 300, 100
|
||||
})
|
||||
})
|
||||
|
|
@ -121,9 +121,9 @@ describe('ColumnStore', () => {
|
|||
const idB = idMapper.getOrAssign('b')
|
||||
const idC = idMapper.getOrAssign('c')
|
||||
|
||||
store.addEntity(idA, { status: 'active' })
|
||||
store.addEntity(idB, { status: 'inactive' })
|
||||
store.addEntity(idC, { status: 'active' })
|
||||
store.addEntity(BigInt(idA), { status: 'active' })
|
||||
store.addEntity(BigInt(idB), { status: 'inactive' })
|
||||
store.addEntity(BigInt(idC), { status: 'active' })
|
||||
await store.flush()
|
||||
|
||||
const bitmap = await store.filter('status', 'active')
|
||||
|
|
@ -133,7 +133,7 @@ describe('ColumnStore', () => {
|
|||
})
|
||||
|
||||
it('returns empty bitmap for no match', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' })
|
||||
await store.flush()
|
||||
|
||||
const bitmap = await store.filter('status', 'deleted')
|
||||
|
|
@ -150,7 +150,7 @@ describe('ColumnStore', () => {
|
|||
const ids: number[] = []
|
||||
for (let i = 0; i < 10; i++) {
|
||||
ids.push(idMapper.getOrAssign(`e${i}`))
|
||||
store.addEntity(ids[i], { price: i * 10 })
|
||||
store.addEntity(BigInt(ids[i]), { price: i * 10 })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
|
|
@ -174,16 +174,16 @@ describe('ColumnStore', () => {
|
|||
const idB = idMapper.getOrAssign('b')
|
||||
const idC = idMapper.getOrAssign('c')
|
||||
|
||||
store.addEntity(idA, { createdAt: 100 })
|
||||
store.addEntity(idB, { createdAt: 200 })
|
||||
store.addEntity(idC, { createdAt: 300 })
|
||||
store.addEntity(BigInt(idA), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idB), { createdAt: 200 })
|
||||
store.addEntity(BigInt(idC), { createdAt: 300 })
|
||||
await store.flush()
|
||||
|
||||
store.removeEntity(idB)
|
||||
store.removeEntity(BigInt(idB))
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['a', 'c'])
|
||||
})
|
||||
})
|
||||
|
|
@ -196,12 +196,12 @@ describe('ColumnStore', () => {
|
|||
it('sorts correctly across multiple L0 segments', async () => {
|
||||
// Flush threshold is 10, so 25 entities creates 3 segments
|
||||
for (let i = 0; i < 25; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`e${i}`), { ts: i })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { ts: i })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('ts', 'desc', 5)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['e24', 'e23', 'e22', 'e21', 'e20'])
|
||||
})
|
||||
})
|
||||
|
|
@ -212,9 +212,9 @@ describe('ColumnStore', () => {
|
|||
|
||||
describe('persistence', () => {
|
||||
it('survives close + reload', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
||||
|
|
@ -223,7 +223,7 @@ describe('ColumnStore', () => {
|
|||
await store2.init(storage, idMapper)
|
||||
|
||||
const result = await store2.sortTopK('createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['a', 'c', 'b'])
|
||||
|
||||
await store2.close()
|
||||
|
|
@ -239,8 +239,8 @@ describe('ColumnStore', () => {
|
|||
const idA = idMapper.getOrAssign('docA')
|
||||
const idB = idMapper.getOrAssign('docB')
|
||||
|
||||
store.addEntity(idA, { __words__: ['machine', 'learning', 'algorithm'] })
|
||||
store.addEntity(idB, { __words__: ['neural', 'network', 'machine'] })
|
||||
store.addEntity(BigInt(idA), { __words__: ['machine', 'learning', 'algorithm'] })
|
||||
store.addEntity(BigInt(idB), { __words__: ['neural', 'network', 'machine'] })
|
||||
await store.flush()
|
||||
|
||||
// Point filter: "machine" should match both
|
||||
|
|
@ -267,10 +267,10 @@ describe('ColumnStore', () => {
|
|||
|
||||
describe('getFilterValues', () => {
|
||||
it('returns distinct values sorted', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'pending' })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('d'), { status: 'inactive' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'pending' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('d')), { status: 'inactive' })
|
||||
await store.flush()
|
||||
|
||||
const values = await store.getFilterValues('status')
|
||||
|
|
@ -288,7 +288,7 @@ describe('ColumnStore', () => {
|
|||
})
|
||||
|
||||
it('returns true after adding data', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 100 })
|
||||
expect(store.hasField('createdAt')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue