feat(8.0): u64 BigInt graph provider contract — punch list a-d,g,h
GraphIndexProvider now speaks BigInt at the boundary (D.2 mirror): - getNeighbors/getVerbIdsBySource/getVerbIdsByTarget take entity ints and return entity/verb ints as bigint[] - new REQUIRED verbIntsToIds(bigint[]) batch reverse resolver (L.7 identity-fingerprint design — verb ids are UUIDs by contract, so the provider-side interning is losslessly reversible) - addVerb(verb, sourceInt, targetInt) returns the interned verb int; removeVerb(verbId) joins the contract Coordinator (brainy.ts) owns ALL UUID <-> int conversion: getOrAssign on writes, getInt on reads (unmapped UUID -> empty result without calling the provider), getUuid / verbIntsToIds on returns, plus a bounded ~100k-entry insertion-order warm cache for verb-int -> verb-id pairs fed by addVerb returns and resolver results. GraphVerb gains derived sourceInt/targetInt (populated at add time, never persisted). findConnectedSubtype gains a native fast path that routes single-type single-subtype outgoing BFS through the provider when available. JS GraphAdjacencyIndex satisfies the contract while staying string/u32-keyed internally: entity ints resolve through the shared entity-id mapper (threaded in by the coordinator on init/fork/checkout), verb ints come from an in-process append-only interning map re-derived from storage on rebuild/cold-start. ColumnStoreProvider widens the same way: addEntity/ removeEntity take bigint, sortTopK/filteredSortTopK return bigint[]. relate() now rejects a caller-supplied id with a teaching error — verb ids are brainy-generated UUIDs by contract in 8.0 (previously a passed id was silently ignored). No Roaring64 provider-boundary decode site exists yet; the JS-internal column store stays Roaring32 and the Treemap decoder lands with the first consumer of provider-returned filter buffers. Public brain API unchanged. 1413 tests green (+10 new BigInt contract tests).
This commit is contained in:
parent
62f6472fa0
commit
2427bb7960
17 changed files with 1164 additions and 328 deletions
306
src/brainy.ts
306
src/brainy.ts
|
|
@ -164,6 +164,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
private distance: DistanceFunction
|
||||
private config: Required<BrainyConfig>
|
||||
|
||||
/**
|
||||
* Bounded warm cache for verb-int → verb-id resolution (8.0 u64 contract).
|
||||
* Fed by `addVerb` returns and `verbIntsToIds` results; evicted in insertion
|
||||
* order once {@link Brainy.VERB_INT_WARM_CACHE_MAX} is exceeded. Pure
|
||||
* optimization — the graph-index provider owns the durable interning, so a
|
||||
* miss just means one extra `verbIntsToIds` round trip.
|
||||
*/
|
||||
private readonly verbIntWarmCache = new Map<bigint, string>()
|
||||
/** Cap for {@link Brainy.verbIntWarmCache} (~100k entries ≈ a few MB). */
|
||||
private static readonly VERB_INT_WARM_CACHE_MAX = 100_000
|
||||
|
||||
// Distributed components (optional)
|
||||
private coordinator?: DistributedCoordinator
|
||||
private shardManager?: ShardManager
|
||||
|
|
@ -602,6 +613,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this.graphIndex = graphIndex
|
||||
}
|
||||
|
||||
// 8.0 u64 contract: thread the shared UUID ↔ int resolver into the JS
|
||||
// graph index and the storage layer's verb read paths.
|
||||
this.wireGraphIdResolver()
|
||||
|
||||
// Wire the connections codec (2.4.0 #3). When the graph:compression
|
||||
// provider is registered AND the metadata index exposes a stable
|
||||
// idMapper, inject a codec that encodes HNSW connections as
|
||||
|
|
@ -1741,9 +1756,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Operations 4+: Delete all related verbs atomically
|
||||
for (const verb of allVerbs) {
|
||||
// Remove from graph index
|
||||
// Remove from graph index (endpoint ints resolved up front so a
|
||||
// rollback can re-add through the BigInt addVerb contract)
|
||||
const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
|
||||
tx.addOperation(
|
||||
new RemoveFromGraphIndexOperation(this.graphIndex, verb as any)
|
||||
new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt)
|
||||
)
|
||||
// Delete verb metadata
|
||||
tx.addOperation(
|
||||
|
|
@ -1768,6 +1785,126 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// ============= RELATIONSHIP OPERATIONS =============
|
||||
|
||||
// --- 8.0 u64 boundary helpers -------------------------------------------
|
||||
// UUID ↔ int conversion happens ONCE here at the coordinator boundary:
|
||||
// `getOrAssign` on writes, `getInt` on reads (undefined → the entity was
|
||||
// never mapped, i.e. it has no relations — return empty without calling the
|
||||
// provider). Provider returns convert back via `getUuid` (entities) and
|
||||
// `verbIntsToIds` + the warm cache (verbs). `Number(bigint)` narrowing is
|
||||
// lossless under the shipped EntityIdSpaceExceeded u32 guard.
|
||||
|
||||
/**
|
||||
* Resolve a UUID to its entity int for a READ — `undefined` means the
|
||||
* entity was never mapped and therefore has no relations.
|
||||
*/
|
||||
private graphEntityInt(uuid: string): bigint | undefined {
|
||||
const intId = this.metadataIndex.getIdMapper().getInt(uuid)
|
||||
return intId === undefined ? undefined : BigInt(intId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a verb's endpoint UUIDs to entity ints for a WRITE
|
||||
* (`getOrAssign`), mirroring them onto `verb.sourceInt`/`verb.targetInt`
|
||||
* (derived state — never persisted to storage JSON) before the verb is
|
||||
* handed to the graph-index provider. Accepts any verb shape carrying
|
||||
* endpoint UUIDs (`GraphVerb`, `HNSWVerbWithMetadata`, …).
|
||||
*/
|
||||
private resolveVerbEndpointInts(
|
||||
verb: Pick<GraphVerb, 'sourceId' | 'targetId'> & { sourceInt?: bigint; targetInt?: bigint }
|
||||
): { sourceInt: bigint; targetInt: bigint } {
|
||||
const idMapper = this.metadataIndex.getIdMapper()
|
||||
const sourceInt = BigInt(idMapper.getOrAssign(verb.sourceId))
|
||||
const targetInt = BigInt(idMapper.getOrAssign(verb.targetId))
|
||||
verb.sourceInt = sourceInt
|
||||
verb.targetInt = targetInt
|
||||
return { sourceInt, targetInt }
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert provider-returned entity ints back to UUIDs, dropping ints the
|
||||
* mapper no longer knows (deleted entities).
|
||||
*/
|
||||
private entityIntsToUuids(entityInts: bigint[]): string[] {
|
||||
const idMapper = this.metadataIndex.getIdMapper()
|
||||
const uuids: string[] = []
|
||||
for (const entityInt of entityInts) {
|
||||
const uuid = idMapper.getUuid(Number(entityInt))
|
||||
if (uuid !== undefined) uuids.push(uuid)
|
||||
}
|
||||
return uuids
|
||||
}
|
||||
|
||||
/**
|
||||
* Record one verb-int → verb-id pair in the bounded warm cache, evicting
|
||||
* the oldest entries (insertion order) past the cap.
|
||||
*/
|
||||
private cacheVerbInt(verbInt: bigint, verbId: string): void {
|
||||
if (!this.verbIntWarmCache.has(verbInt) &&
|
||||
this.verbIntWarmCache.size >= Brainy.VERB_INT_WARM_CACHE_MAX) {
|
||||
// Evict oldest insertions until under cap (single eviction in the
|
||||
// common case; loop guards against future cap reductions).
|
||||
for (const oldest of this.verbIntWarmCache.keys()) {
|
||||
this.verbIntWarmCache.delete(oldest)
|
||||
if (this.verbIntWarmCache.size < Brainy.VERB_INT_WARM_CACHE_MAX) break
|
||||
}
|
||||
}
|
||||
this.verbIntWarmCache.set(verbInt, verbId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve provider-returned verb ints to verb-id strings: warm cache first,
|
||||
* then one batched `verbIntsToIds` call for the misses (which also refills
|
||||
* the cache). Unknown ints are dropped.
|
||||
*/
|
||||
private async resolveVerbIntsToIds(verbInts: bigint[]): Promise<string[]> {
|
||||
if (verbInts.length === 0) return []
|
||||
|
||||
const resolved = new Array<string | undefined>(verbInts.length)
|
||||
const missIndices: number[] = []
|
||||
const missInts: bigint[] = []
|
||||
|
||||
for (let i = 0; i < verbInts.length; i++) {
|
||||
const cached = this.verbIntWarmCache.get(verbInts[i])
|
||||
if (cached !== undefined) {
|
||||
resolved[i] = cached
|
||||
} else {
|
||||
missIndices.push(i)
|
||||
missInts.push(verbInts[i])
|
||||
}
|
||||
}
|
||||
|
||||
if (missInts.length > 0) {
|
||||
const ids = await this.graphIndex.verbIntsToIds(missInts)
|
||||
for (let j = 0; j < missInts.length; j++) {
|
||||
const id = ids[j]
|
||||
if (id !== null) {
|
||||
resolved[missIndices[j]] = id
|
||||
this.cacheVerbInt(missInts[j], id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return resolved.filter((id): id is string => id !== undefined)
|
||||
}
|
||||
|
||||
/**
|
||||
* UUID-level neighbor lookup over the BigInt provider contract: resolves
|
||||
* the anchor via `getInt` (unmapped → empty), then maps returned entity
|
||||
* ints back to UUIDs. Shared by the traversal paths and the
|
||||
* TripleIntelligenceSystem adapter.
|
||||
*/
|
||||
private async getNeighborUuids(
|
||||
uuid: string,
|
||||
options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number }
|
||||
): Promise<string[]> {
|
||||
const entityInt = this.graphEntityInt(uuid)
|
||||
if (entityInt === undefined) return []
|
||||
const neighborInts = await this.graphIndex.getNeighbors(entityInt, options)
|
||||
return this.entityIntsToUuids(neighborInts)
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Create a relationship (verb) between two entities
|
||||
*
|
||||
|
|
@ -1790,7 +1927,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* @param params.service - Multi-tenancy service name
|
||||
* @param params.confidence - Relationship certainty 0-1
|
||||
* @param params.evidence - Why this relationship exists
|
||||
* @returns Promise that resolves to the relationship ID
|
||||
* @returns Promise that resolves to the relationship ID. **Contract (8.0):**
|
||||
* relationship ids are always UUIDs (36-char 8-4-4-4-12 hex). Brainy
|
||||
* generates every verb id itself (`relate()` takes no custom id), so the
|
||||
* UUID shape is a guarantee, not a coincidence — graph-index providers may
|
||||
* key their verb-int interning on the raw UUID bytes for lossless
|
||||
* reverse lookup.
|
||||
*
|
||||
* @example
|
||||
* // Basic relationship creation
|
||||
|
|
@ -1918,7 +2060,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// This prevents infinite loops where same relationship is created repeatedly
|
||||
// Bug #1 showed incrementing verb counts (7→8→9...) indicating duplicates
|
||||
// OPTIMIZATION: Use GraphAdjacencyIndex for O(log n) lookup instead of O(n) storage scan
|
||||
const verbIds = await this.graphIndex.getVerbIdsBySource(params.from)
|
||||
// 8.0 BigInt boundary: unmapped source → no existing relations to check.
|
||||
const dupSourceInt = this.graphEntityInt(params.from)
|
||||
const verbIds = dupSourceInt === undefined
|
||||
? []
|
||||
: await this.resolveVerbIntsToIds(await this.graphIndex.getVerbIdsBySource(dupSourceInt))
|
||||
|
||||
// Batch-load verbs for 5x faster duplicate checking on GCS
|
||||
// GCS: 5 verbs = 1×50ms vs 5×50ms = 250ms (5x faster)
|
||||
|
|
@ -1969,6 +2115,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
createdAt: Date.now()
|
||||
}
|
||||
|
||||
// 8.0 BigInt boundary: resolve endpoint ints ONCE (getOrAssign — both
|
||||
// entities were existence-checked above) and mirror them onto the verb.
|
||||
const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
|
||||
|
||||
// Execute atomically with transaction system
|
||||
await this.transactionManager.executeTransaction(async (tx) => {
|
||||
// Operation 1: Save verb vector data
|
||||
|
|
@ -1990,7 +2140,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Operation 3: Add to graph index for O(1) lookups
|
||||
tx.addOperation(
|
||||
new AddToGraphIndexOperation(this.graphIndex, verb)
|
||||
new AddToGraphIndexOperation(
|
||||
this.graphIndex, verb, sourceInt, targetInt,
|
||||
(verbInt) => this.cacheVerbInt(verbInt, id)
|
||||
)
|
||||
)
|
||||
|
||||
// Create bidirectional if requested
|
||||
|
|
@ -2000,7 +2153,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
...verb,
|
||||
id: reverseId,
|
||||
sourceId: params.to,
|
||||
targetId: params.from
|
||||
targetId: params.from,
|
||||
// Endpoints swap, so the derived ints swap with them.
|
||||
sourceInt: targetInt,
|
||||
targetInt: sourceInt
|
||||
}
|
||||
|
||||
// Operation 4: Save reverse verb vector data
|
||||
|
|
@ -2022,7 +2178,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Operation 6: Add reverse relationship to graph index
|
||||
tx.addOperation(
|
||||
new AddToGraphIndexOperation(this.graphIndex, reverseVerb)
|
||||
new AddToGraphIndexOperation(
|
||||
this.graphIndex, reverseVerb, targetInt, sourceInt,
|
||||
(verbInt) => this.cacheVerbInt(verbInt, reverseId)
|
||||
)
|
||||
)
|
||||
}
|
||||
})
|
||||
|
|
@ -2053,12 +2212,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Get verb data before deletion for rollback
|
||||
const verb = await this.storage.getVerb(id)
|
||||
|
||||
// 8.0 BigInt boundary: resolve endpoint ints before the transaction so
|
||||
// a rollback can re-add through the BigInt addVerb contract.
|
||||
const endpointInts = verb ? this.resolveVerbEndpointInts(verb) : undefined
|
||||
|
||||
// Execute atomically with transaction system
|
||||
await this.transactionManager.executeTransaction(async (tx) => {
|
||||
// Operation 1: Remove from graph index
|
||||
if (verb) {
|
||||
if (verb && endpointInts) {
|
||||
tx.addOperation(
|
||||
new RemoveFromGraphIndexOperation(this.graphIndex, verb as any)
|
||||
new RemoveFromGraphIndexOperation(
|
||||
this.graphIndex, verb, endpointInts.sourceInt, endpointInts.targetInt
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -2158,6 +2323,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
createdAt: existingAny.createdAt
|
||||
}
|
||||
|
||||
// 8.0 BigInt boundary: endpoints are unchanged across a type swap, so one
|
||||
// resolution serves both the remove (rollback re-add) and the re-add.
|
||||
const reindexInts = typeChanged ? this.resolveVerbEndpointInts(verbForIndex) : undefined
|
||||
|
||||
await this.transactionManager.executeTransaction(async (tx) => {
|
||||
tx.addOperation(
|
||||
new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata)
|
||||
|
|
@ -2165,12 +2334,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// If the verb type changed, re-index in graph adjacency so traversal-by-type
|
||||
// stays consistent. The id is preserved across the swap.
|
||||
if (typeChanged) {
|
||||
if (typeChanged && reindexInts) {
|
||||
tx.addOperation(
|
||||
new RemoveFromGraphIndexOperation(this.graphIndex, existing as any)
|
||||
new RemoveFromGraphIndexOperation(
|
||||
this.graphIndex, existing, reindexInts.sourceInt, reindexInts.targetInt
|
||||
)
|
||||
)
|
||||
tx.addOperation(
|
||||
new AddToGraphIndexOperation(this.graphIndex, verbForIndex)
|
||||
new AddToGraphIndexOperation(
|
||||
this.graphIndex, verbForIndex, reindexInts.sourceInt, reindexInts.targetInt,
|
||||
(verbInt) => this.cacheVerbInt(verbInt, params.id)
|
||||
)
|
||||
)
|
||||
}
|
||||
})
|
||||
|
|
@ -3027,10 +3201,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
k
|
||||
)
|
||||
|
||||
// Convert int IDs to UUIDs and paginate
|
||||
// Convert int IDs (BigInt at the provider boundary) to UUIDs and
|
||||
// paginate. Number() narrowing is lossless under the u32 guard.
|
||||
const idMapper = this.metadataIndex.getIdMapper()
|
||||
const allUuids = sortedIntIds
|
||||
.map(intId => idMapper.getUuid(intId))
|
||||
.map(intId => idMapper.getUuid(Number(intId)))
|
||||
.filter((uuid): uuid is string => uuid !== undefined)
|
||||
const pageIds = allUuids.slice(offset, offset + limit)
|
||||
|
||||
|
|
@ -3759,8 +3934,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
|
||||
for (const verb of allVerbs) {
|
||||
const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb)
|
||||
tx.addOperation(
|
||||
new RemoveFromGraphIndexOperation(this.graphIndex, verb as any)
|
||||
new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt)
|
||||
)
|
||||
tx.addOperation(
|
||||
new DeleteVerbMetadataOperation(this.storage, verb.id)
|
||||
|
|
@ -4218,6 +4394,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
// getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex)
|
||||
|
||||
// 8.0 u64 contract: the clone has its own metadata index (and idMapper),
|
||||
// so thread the clone's resolver into its graph index + storage.
|
||||
clone.wireGraphIdResolver()
|
||||
|
||||
// Mark as initialized
|
||||
clone.initialized = true
|
||||
clone.dimensions = this.dimensions
|
||||
|
|
@ -4318,6 +4498,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this.graphIndex = await (this.storage as any).getGraphIndex()
|
||||
}
|
||||
|
||||
// 8.0 u64 contract: the branch switch recreated the metadata index (and
|
||||
// idMapper) plus the graph index — re-thread the shared resolver.
|
||||
this.wireGraphIdResolver()
|
||||
|
||||
// Reset lazy loading state when switching branches
|
||||
// Indexes contain data from previous branch, must rebuild for new branch
|
||||
this.lazyRebuildCompleted = false
|
||||
|
|
@ -5697,11 +5881,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*/
|
||||
getTripleIntelligence(): TripleIntelligenceSystem {
|
||||
if (!this._tripleIntelligence) {
|
||||
// Use core components directly - no lazy loading needed
|
||||
// Use core components directly - no lazy loading needed.
|
||||
// TripleIntelligenceSystem speaks UUIDs; adapt the BigInt graph-index
|
||||
// contract through the coordinator's UUID-level helper so the int
|
||||
// conversion stays at this boundary.
|
||||
this._tripleIntelligence = new TripleIntelligenceSystem(
|
||||
this.metadataIndex,
|
||||
this.index,
|
||||
this.graphIndex,
|
||||
{
|
||||
getNeighbors: (id: string, direction?: 'in' | 'out' | 'both') =>
|
||||
this.getNeighborUuids(id, { direction }),
|
||||
size: () => this.graphIndex.size()
|
||||
},
|
||||
async (text: string) => this.embedder(text),
|
||||
this.storage
|
||||
)
|
||||
|
|
@ -7917,13 +8108,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
verbTypes?: Set<VerbType>,
|
||||
limit?: number
|
||||
): Promise<string[]> {
|
||||
const neighbors = await this.graphIndex.getNeighbors(nodeId, { direction: graphDirection, limit })
|
||||
// 8.0 BigInt boundary: unmapped node → no relations.
|
||||
const nodeInt = this.graphEntityInt(nodeId)
|
||||
if (nodeInt === undefined) return []
|
||||
|
||||
const neighbors = this.entityIntsToUuids(
|
||||
await this.graphIndex.getNeighbors(nodeInt, { direction: graphDirection, limit })
|
||||
)
|
||||
if (!verbTypes || verbTypes.size === 0) return neighbors
|
||||
|
||||
// Gather candidate edges in the relevant direction(s).
|
||||
const verbIds: string[] = []
|
||||
if (graphDirection !== 'in') verbIds.push(...await this.graphIndex.getVerbIdsBySource(nodeId))
|
||||
if (graphDirection !== 'out') verbIds.push(...await this.graphIndex.getVerbIdsByTarget(nodeId))
|
||||
const verbInts: bigint[] = []
|
||||
if (graphDirection !== 'in') verbInts.push(...await this.graphIndex.getVerbIdsBySource(nodeInt))
|
||||
if (graphDirection !== 'out') verbInts.push(...await this.graphIndex.getVerbIdsByTarget(nodeInt))
|
||||
const verbIds = await this.resolveVerbIntsToIds(verbInts)
|
||||
|
||||
const verbs = await this.graphIndex.getVerbsBatchCached(verbIds)
|
||||
const neighborSet = new Set(neighbors)
|
||||
|
|
@ -8397,14 +8595,52 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
if (subtypeFilter !== undefined) {
|
||||
// Multi-hop BFS with per-hop (verbType, subtype) predicate. Works on the
|
||||
// open-core JS path at any depth. If a graph-index provider exposes a
|
||||
// open-core JS path at any depth. When a graph-index provider exposes a
|
||||
// faster `findConnectedSubtype` (native BFS with the same semantics),
|
||||
// we'll route through it transparently — same pattern as every other
|
||||
// provider hook.
|
||||
// `nativeSubtypeBfs` routes through it transparently — same pattern as
|
||||
// every other provider hook.
|
||||
const subtypeArr = Array.isArray(subtypeFilter) ? subtypeFilter : [subtypeFilter]
|
||||
const subtypeSet = new Set(subtypeArr)
|
||||
|
||||
// Native fast path (D.3): single verb type + single subtype + outgoing
|
||||
// walk match the native BFS semantics exactly (out-edges only, source
|
||||
// excluded, visited-set cycle guard). Entity ints in, entity ints out —
|
||||
// UUID conversion stays at this boundary. Returns null when the query
|
||||
// shape (or provider) can't take the native route.
|
||||
const nativeSubtypeBfs = async (
|
||||
anchor: string,
|
||||
walk: 'in' | 'out' | 'both'
|
||||
): Promise<Set<string> | null> => {
|
||||
if (walk !== 'out') return null
|
||||
if (via === undefined || Array.isArray(via)) return null
|
||||
if (subtypeArr.length !== 1) return null
|
||||
const provider = this.graphIndex as Partial<{
|
||||
findConnectedSubtype(
|
||||
sourceInt: bigint,
|
||||
verbTypeIndex: number,
|
||||
subtype: string | null,
|
||||
depth: number,
|
||||
limit?: number | null
|
||||
): Promise<bigint[]>
|
||||
}>
|
||||
if (typeof provider.findConnectedSubtype !== 'function') return null
|
||||
|
||||
const anchorInt = this.graphEntityInt(anchor)
|
||||
if (anchorInt === undefined) return new Set() // unmapped → no relations
|
||||
|
||||
const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType)
|
||||
// No limit: match the JS BFS exactly — overall result limiting happens
|
||||
// downstream against existingResults.
|
||||
const reachedInts = await provider.findConnectedSubtype(
|
||||
anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null
|
||||
)
|
||||
return new Set(this.entityIntsToUuids(reachedInts))
|
||||
}
|
||||
|
||||
const bfsWithSubtype = async (anchor: string, walk: 'in' | 'out' | 'both'): Promise<Set<string>> => {
|
||||
const nativeResult = await nativeSubtypeBfs(anchor, walk)
|
||||
if (nativeResult !== null) return nativeResult
|
||||
|
||||
const visited = new Set<string>([anchor])
|
||||
const reached = new Set<string>()
|
||||
let frontier = new Set<string>([anchor])
|
||||
|
|
@ -8730,13 +8966,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
if (constraints.to) {
|
||||
// Check if this entity connects TO the target (O(1) lookup)
|
||||
const outgoingNeighbors = await this.graphIndex.getNeighbors(result.id, 'out')
|
||||
const outgoingNeighbors = await this.getNeighborUuids(result.id, { direction: 'out' })
|
||||
hasConnection = outgoingNeighbors.includes(constraints.to)
|
||||
}
|
||||
|
||||
if (constraints.from && !hasConnection) {
|
||||
// Check if this entity connects FROM the source (O(1) lookup)
|
||||
const incomingNeighbors = await this.graphIndex.getNeighbors(result.id, 'in')
|
||||
const incomingNeighbors = await this.getNeighborUuids(result.id, { direction: 'in' })
|
||||
hasConnection = incomingNeighbors.includes(constraints.from)
|
||||
}
|
||||
|
||||
|
|
@ -9368,6 +9604,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
await this.lazyRebuildPromise
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Wire the shared UUID ↔ int resolver (the metadata index's
|
||||
* idMapper) into the JS graph index and the storage layer (8.0 u64
|
||||
* contract). Called after the graph index is resolved on init, fork, and
|
||||
* checkout so every consumer of the BigInt boundary shares one int
|
||||
* universe. Native graph-index providers carry their own mapper and don't
|
||||
* expose `setEntityIdMapper` — the storage-level wiring still applies so
|
||||
* its verb read paths can convert UUIDs before provider calls.
|
||||
*/
|
||||
private wireGraphIdResolver(): void {
|
||||
const resolver = this.metadataIndex.getIdMapper()
|
||||
this.storage.setGraphEntityIdResolver(resolver)
|
||||
const jsGraphIndex = this.graphIndex as Partial<Pick<GraphAdjacencyIndex, 'setEntityIdMapper'>>
|
||||
if (typeof jsGraphIndex.setEntityIdMapper === 'function') {
|
||||
jsGraphIndex.setEntityIdMapper(resolver)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Wire the HNSW connections codec (2.4.0 #3). Activates when
|
||||
* BOTH (a) the `graph:compression` provider is registered (cortex registers
|
||||
|
|
|
|||
|
|
@ -420,7 +420,7 @@ export interface HNSWVerbWithMetadata {
|
|||
* `relate()` / `getRelations()` surface, and storage adapters all speak it.
|
||||
*/
|
||||
export interface GraphVerb {
|
||||
id: string // Unique identifier for the verb
|
||||
id: string // Unique identifier — always a UUID (8.0 contract: brainy generates every verb id, so providers may key verb-int interning on the raw UUID bytes)
|
||||
sourceId: string // ID of the source noun
|
||||
targetId: string // ID of the target noun
|
||||
vector: Vector // Vector representation of the relationship
|
||||
|
|
@ -439,6 +439,22 @@ export interface GraphVerb {
|
|||
createdAt?: number | { seconds: number; nanoseconds: number } // When the verb was created
|
||||
updatedAt?: number | { seconds: number; nanoseconds: number } // When the verb was last updated
|
||||
createdBy?: { augmentation: string; version: string } // Information about what created this verb
|
||||
|
||||
/**
|
||||
* Derived state: the source entity's interned integer ID (u64-safe BigInt).
|
||||
* Populated by the coordinator (`brainy.ts`) from the shared entity-id mapper
|
||||
* at add time, immediately before the verb is handed to the graph-index
|
||||
* provider. NEVER persisted to storage JSON — the mapper is the source of
|
||||
* truth, and persisting would denormalize state that can go stale across a
|
||||
* mapper rebuild.
|
||||
*/
|
||||
sourceInt?: bigint
|
||||
/**
|
||||
* Derived state: the target entity's interned integer ID (u64-safe BigInt).
|
||||
* Same lifecycle as {@link GraphVerb.sourceInt}: coordinator-populated at add
|
||||
* time, never persisted to storage JSON.
|
||||
*/
|
||||
targetInt?: bigint
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,11 +1,20 @@
|
|||
/**
|
||||
* GraphAdjacencyIndex - Billion-Scale Graph Traversal Engine
|
||||
* @module graph/graphAdjacencyIndex
|
||||
* @description GraphAdjacencyIndex — billion-scale graph traversal engine.
|
||||
*
|
||||
* NOW SCALES TO BILLIONS: LSM-tree storage reduces memory from 500GB to 1.3GB
|
||||
* for 1 billion relationships while maintaining sub-5ms neighbor lookups.
|
||||
* LSM-tree storage reduces memory from 500GB to 1.3GB for 1 billion
|
||||
* relationships while maintaining sub-5ms neighbor lookups.
|
||||
*
|
||||
* NO FALLBACKS - NO MOCKS - REAL PRODUCTION CODE
|
||||
* Handles billions of relationships with sustainable memory usage
|
||||
* **8.0 u64 boundary:** this class implements the BigInt
|
||||
* {@link GraphIndexProvider} contract — entity ints in, entity/verb ints out.
|
||||
* Internally everything stays string-keyed (UUID-keyed LSM trees) and u32:
|
||||
* entity-int params resolve to UUIDs via the shared entity-id mapper
|
||||
* (`getUuid(Number(big))` — lossless under the `EntityIdSpaceExceeded` u32
|
||||
* guard), and returns convert back with `BigInt(getOrAssign(uuid))`. Verb ints
|
||||
* come from a small in-process interning map (see
|
||||
* {@link GraphAdjacencyIndex.verbIntsToIds}); the interning is derived state,
|
||||
* never persisted — `rebuild()` / the verb-id-set recovery path re-derive it
|
||||
* from storage, which the JS index already does on cold start.
|
||||
*/
|
||||
|
||||
import { GraphVerb, StorageAdapter } from '../coreTypes.js'
|
||||
|
|
@ -21,6 +30,22 @@ export interface GraphIndexConfig {
|
|||
flushInterval?: number // Default: 30000ms
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The minimal UUID ↔ int resolver surface the JS graph index
|
||||
* needs at its BigInt boundary. Satisfied by `EntityIdMapper` (and by whatever
|
||||
* `MetadataIndexManager.getIdMapper()` / a native metadata index returns) —
|
||||
* declared structurally here so the graph layer doesn't import the metadata
|
||||
* layer.
|
||||
*/
|
||||
export interface GraphEntityIdResolver {
|
||||
/** Resolve a UUID to its int, assigning a new one if absent (write path). */
|
||||
getOrAssign(uuid: string): number
|
||||
/** Resolve a UUID to its int without assigning (read path). */
|
||||
getInt(uuid: string): number | undefined
|
||||
/** Reverse-resolve an int to its UUID (`undefined` = unknown/deleted). */
|
||||
getUuid(intId: number): string | undefined
|
||||
}
|
||||
|
||||
export interface GraphIndexStats {
|
||||
totalRelationships: number
|
||||
sourceNodes: number
|
||||
|
|
@ -51,6 +76,18 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
// Now: Set<string> stores only IDs (~100KB @ 1B verbs) = 1,280,000x reduction
|
||||
private verbIdSet = new Set<string>()
|
||||
|
||||
// Verb-id interning for the BigInt boundary (8.0 u64 contract).
|
||||
// Process-lifetime derived state: assigned on addVerb / rebuild /
|
||||
// verb-id-set recovery, NEVER persisted. removeVerb keeps the entry so a
|
||||
// verb int stays stable (and resolvable) for the index's lifetime.
|
||||
private verbIdToInt = new Map<string, number>()
|
||||
private verbIntToId: string[] = []
|
||||
|
||||
// Shared UUID ↔ int resolver for entity ints at the BigInt boundary.
|
||||
// Threaded in by the coordinator (brainy.ts) from the metadata index's
|
||||
// idMapper — see setEntityIdMapper().
|
||||
private entityIdMapper?: GraphEntityIdResolver
|
||||
|
||||
// Infrastructure integration
|
||||
private storage: StorageAdapter
|
||||
private unifiedCache: UnifiedCache
|
||||
|
|
@ -75,8 +112,13 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
return this.initialized
|
||||
}
|
||||
|
||||
constructor(storage: StorageAdapter, config: GraphIndexConfig = {}) {
|
||||
constructor(
|
||||
storage: StorageAdapter,
|
||||
config: GraphIndexConfig = {},
|
||||
entityIdMapper?: GraphEntityIdResolver
|
||||
) {
|
||||
this.storage = storage
|
||||
this.entityIdMapper = entityIdMapper
|
||||
this.config = {
|
||||
maxIndexSize: config.maxIndexSize ?? 100000,
|
||||
rebuildThreshold: config.rebuildThreshold ?? 0.1,
|
||||
|
|
@ -116,6 +158,49 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
prodLog.info('GraphAdjacencyIndex initialized with LSM-tree storage (4 LSM-trees total)')
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Thread in the shared UUID ↔ int resolver used for entity-int
|
||||
* conversion at the BigInt boundary. The coordinator (`brainy.ts`) calls this
|
||||
* with `metadataIndex.getIdMapper()` right after index construction (init,
|
||||
* fork, and checkout paths) so reads/writes share one int universe with the
|
||||
* metadata index. Idempotent; safe to call again after a branch switch.
|
||||
* @param mapper - The shared entity-id resolver.
|
||||
* @returns Nothing.
|
||||
*/
|
||||
setEntityIdMapper(mapper: GraphEntityIdResolver): void {
|
||||
this.entityIdMapper = mapper
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the entity-id mapper or fail loudly. The BigInt read methods are
|
||||
* meaningless without a shared int universe — a missing mapper is a wiring
|
||||
* bug, not a recoverable condition.
|
||||
*/
|
||||
private requireEntityIdMapper(): GraphEntityIdResolver {
|
||||
if (!this.entityIdMapper) {
|
||||
throw new Error(
|
||||
'GraphAdjacencyIndex: entityIdMapper not wired. The coordinator must ' +
|
||||
'call setEntityIdMapper(metadataIndex.getIdMapper()) (or pass it to ' +
|
||||
'the constructor) before BigInt-boundary reads.'
|
||||
)
|
||||
}
|
||||
return this.entityIdMapper
|
||||
}
|
||||
|
||||
/**
|
||||
* Intern a verb-id string, assigning the next sequential u32 on first sight.
|
||||
* Append-only for the index's lifetime — removeVerb keeps the entry so verb
|
||||
* ints handed to callers stay resolvable.
|
||||
*/
|
||||
private internVerbId(verbId: string): number {
|
||||
const existing = this.verbIdToInt.get(verbId)
|
||||
if (existing !== undefined) return existing
|
||||
const next = this.verbIntToId.length
|
||||
this.verbIdToInt.set(verbId, next)
|
||||
this.verbIntToId.push(verbId)
|
||||
return next
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the graph index (lazy initialization)
|
||||
* Added defensive auto-rebuild check for verbIdSet consistency
|
||||
|
|
@ -172,6 +257,10 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
|
||||
for (const verb of result.items) {
|
||||
this.verbIdSet.add(verb.id)
|
||||
// Re-derive the verb-int interning (process-lifetime state, never
|
||||
// persisted — this recovery path is one of the two cold-start sources,
|
||||
// alongside rebuild()).
|
||||
this.internVerbId(verb.id)
|
||||
// Also update counts
|
||||
const verbType = verb.verb || 'unknown'
|
||||
this.relationshipCountsByType.set(
|
||||
|
|
@ -190,49 +279,60 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
}
|
||||
|
||||
/**
|
||||
* Core API - Neighbor lookup with LSM-tree storage
|
||||
* @description Core API — neighbor lookup with LSM-tree storage (BigInt
|
||||
* boundary). O(log n) with bloom filter optimization (90% of queries skip
|
||||
* disk I/O); pagination support for high-degree nodes. The entity int is
|
||||
* resolved to a UUID via the shared mapper (unknown int → empty result) and
|
||||
* neighbor UUIDs convert back via `BigInt(getOrAssign(uuid))`.
|
||||
*
|
||||
* O(log n) with bloom filter optimization (90% of queries skip disk I/O)
|
||||
* Added pagination support for high-degree nodes
|
||||
*
|
||||
* @param id Entity ID to get neighbors for
|
||||
* @param optionsOrDirection Optional: direction string OR options object
|
||||
* @returns Array of neighbor IDs (paginated if limit/offset specified)
|
||||
* @param id - Entity int to get neighbors for (from the shared idMapper).
|
||||
* @param options - Optional direction ('both' default) + limit/offset.
|
||||
* @returns Neighbor entity ints (paginated if limit/offset specified).
|
||||
*
|
||||
* @example
|
||||
* // Get all neighbors (backward compatible)
|
||||
* const all = await graphIndex.getNeighbors(id)
|
||||
* // Get all neighbors of an entity int
|
||||
* const all = await graphIndex.getNeighbors(42n)
|
||||
*
|
||||
* @example
|
||||
* // Get outgoing neighbors (backward compatible)
|
||||
* const out = await graphIndex.getNeighbors(id, 'out')
|
||||
*
|
||||
* @example
|
||||
* // Get first 50 outgoing neighbors (new API)
|
||||
* const page1 = await graphIndex.getNeighbors(id, { direction: 'out', limit: 50 })
|
||||
*
|
||||
* @example
|
||||
* // Paginate through neighbors
|
||||
* const page1 = await graphIndex.getNeighbors(id, { limit: 100, offset: 0 })
|
||||
* const page2 = await graphIndex.getNeighbors(id, { limit: 100, offset: 100 })
|
||||
* // Get first 50 outgoing neighbors
|
||||
* const page1 = await graphIndex.getNeighbors(42n, { direction: 'out', limit: 50 })
|
||||
*/
|
||||
async getNeighbors(
|
||||
id: string,
|
||||
optionsOrDirection?: {
|
||||
id: bigint,
|
||||
options?: {
|
||||
direction?: 'in' | 'out' | 'both'
|
||||
limit?: number
|
||||
offset?: number
|
||||
} | 'in' | 'out' | 'both'
|
||||
): Promise<string[]> {
|
||||
}
|
||||
): Promise<bigint[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Normalize old API (direction string) to new API (options object)
|
||||
const options = typeof optionsOrDirection === 'string'
|
||||
? { direction: optionsOrDirection }
|
||||
: (optionsOrDirection || {})
|
||||
const mapper = this.requireEntityIdMapper()
|
||||
const uuid = mapper.getUuid(Number(id))
|
||||
if (uuid === undefined) {
|
||||
// Unknown/deleted entity int — no edges by definition.
|
||||
return []
|
||||
}
|
||||
|
||||
const neighborUuids = await this.getNeighborUuids(uuid, options)
|
||||
return neighborUuids.map(neighborUuid => BigInt(mapper.getOrAssign(neighborUuid)))
|
||||
}
|
||||
|
||||
/**
|
||||
* String-keyed neighbor lookup — the internal implementation behind
|
||||
* {@link getNeighbors}. Kept UUID-based because the LSM trees are keyed by
|
||||
* UUID; only the public contract speaks BigInt.
|
||||
*/
|
||||
private async getNeighborUuids(
|
||||
id: string,
|
||||
options?: {
|
||||
direction?: 'in' | 'out' | 'both'
|
||||
limit?: number
|
||||
offset?: number
|
||||
}
|
||||
): Promise<string[]> {
|
||||
const startTime = performance.now()
|
||||
const direction = options.direction || 'both'
|
||||
const direction = options?.direction || 'both'
|
||||
const neighbors = new Set<string>()
|
||||
|
||||
// Query LSM-trees with bloom filter optimization
|
||||
|
|
@ -255,8 +355,8 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
|
||||
// Apply pagination if requested
|
||||
if (options?.limit !== undefined || options?.offset !== undefined) {
|
||||
const offset = options.offset || 0
|
||||
const limit = options.limit !== undefined ? options.limit : result.length
|
||||
const offset = options?.offset || 0
|
||||
const limit = options?.limit !== undefined ? options.limit : result.length
|
||||
result = result.slice(offset, offset + limit)
|
||||
}
|
||||
|
||||
|
|
@ -271,40 +371,34 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get verb IDs by source - Billion-scale optimization for getVerbsBySource
|
||||
* @description Verb ints for all edges originating at `sourceInt` (BigInt
|
||||
* boundary). O(log n) LSM-tree lookup with bloom filter optimization;
|
||||
* filters out deleted verb IDs (tombstone deletion workaround); pagination
|
||||
* support for entities with many relationships. Unknown entity int → empty
|
||||
* result. Resolve returned ints back to verb-id strings with
|
||||
* {@link verbIntsToIds}.
|
||||
*
|
||||
* O(log n) LSM-tree lookup with bloom filter optimization
|
||||
* Filters out deleted verb IDs (tombstone deletion workaround)
|
||||
* Added pagination support for entities with many relationships
|
||||
*
|
||||
* @param sourceId Source entity ID
|
||||
* @param options Optional configuration
|
||||
* @param options.limit Maximum number of verb IDs to return (default: all)
|
||||
* @param options.offset Number of verb IDs to skip (default: 0)
|
||||
* @returns Array of verb IDs originating from this source (excluding deleted, paginated if requested)
|
||||
* @param sourceInt - Source entity int (from the shared idMapper).
|
||||
* @param options - Optional limit/offset pagination.
|
||||
* @returns Verb ints originating from this source (excluding deleted).
|
||||
*
|
||||
* @example
|
||||
* // Get all verb IDs (backward compatible)
|
||||
* const all = await graphIndex.getVerbIdsBySource(sourceId)
|
||||
*
|
||||
* @example
|
||||
* // Get first 50 verb IDs
|
||||
* const page1 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 50 })
|
||||
*
|
||||
* @example
|
||||
* // Paginate through verb IDs
|
||||
* const page1 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 100, offset: 0 })
|
||||
* const page2 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 100, offset: 100 })
|
||||
* const verbInts = await graphIndex.getVerbIdsBySource(42n, { limit: 50 })
|
||||
* const verbIds = await graphIndex.verbIntsToIds(verbInts)
|
||||
*/
|
||||
async getVerbIdsBySource(
|
||||
sourceId: string,
|
||||
sourceInt: bigint,
|
||||
options?: {
|
||||
limit?: number
|
||||
offset?: number
|
||||
}
|
||||
): Promise<string[]> {
|
||||
): Promise<bigint[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const mapper = this.requireEntityIdMapper()
|
||||
const sourceId = mapper.getUuid(Number(sourceInt))
|
||||
if (sourceId === undefined) return []
|
||||
|
||||
const startTime = performance.now()
|
||||
const verbIds = await this.lsmTreeVerbsBySource.get(sourceId)
|
||||
const elapsed = performance.now() - startTime
|
||||
|
|
@ -314,56 +408,37 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
prodLog.warn(`GraphAdjacencyIndex: Slow getVerbIdsBySource for ${sourceId}: ${elapsed.toFixed(2)}ms`)
|
||||
}
|
||||
|
||||
// Filter out deleted verb IDs (tombstone deletion workaround)
|
||||
// LSM-tree retains all IDs, but verbIdSet tracks deletions
|
||||
const allIds = verbIds || []
|
||||
let result = allIds.filter(id => this.verbIdSet.has(id))
|
||||
|
||||
// Apply pagination if requested
|
||||
if (options?.limit !== undefined || options?.offset !== undefined) {
|
||||
const offset = options.offset || 0
|
||||
const limit = options.limit !== undefined ? options.limit : result.length
|
||||
result = result.slice(offset, offset + limit)
|
||||
}
|
||||
|
||||
return result
|
||||
return this.verbIdsToPaginatedInts(verbIds || [], options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get verb IDs by target - Billion-scale optimization for getVerbsByTarget
|
||||
* @description Verb ints for all edges pointing at `targetInt` (BigInt
|
||||
* boundary). O(log n) LSM-tree lookup with bloom filter optimization;
|
||||
* filters out deleted verb IDs (tombstone deletion workaround); pagination
|
||||
* support for popular target entities. Unknown entity int → empty result.
|
||||
* Resolve returned ints back to verb-id strings with {@link verbIntsToIds}.
|
||||
*
|
||||
* O(log n) LSM-tree lookup with bloom filter optimization
|
||||
* Filters out deleted verb IDs (tombstone deletion workaround)
|
||||
* Added pagination support for popular target entities
|
||||
*
|
||||
* @param targetId Target entity ID
|
||||
* @param options Optional configuration
|
||||
* @param options.limit Maximum number of verb IDs to return (default: all)
|
||||
* @param options.offset Number of verb IDs to skip (default: 0)
|
||||
* @returns Array of verb IDs pointing to this target (excluding deleted, paginated if requested)
|
||||
* @param targetInt - Target entity int (from the shared idMapper).
|
||||
* @param options - Optional limit/offset pagination.
|
||||
* @returns Verb ints pointing to this target (excluding deleted).
|
||||
*
|
||||
* @example
|
||||
* // Get all verb IDs (backward compatible)
|
||||
* const all = await graphIndex.getVerbIdsByTarget(targetId)
|
||||
*
|
||||
* @example
|
||||
* // Get first 50 verb IDs
|
||||
* const page1 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 50 })
|
||||
*
|
||||
* @example
|
||||
* // Paginate through verb IDs
|
||||
* const page1 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 100, offset: 0 })
|
||||
* const page2 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 100, offset: 100 })
|
||||
* const verbInts = await graphIndex.getVerbIdsByTarget(42n, { limit: 50 })
|
||||
* const verbIds = await graphIndex.verbIntsToIds(verbInts)
|
||||
*/
|
||||
async getVerbIdsByTarget(
|
||||
targetId: string,
|
||||
targetInt: bigint,
|
||||
options?: {
|
||||
limit?: number
|
||||
offset?: number
|
||||
}
|
||||
): Promise<string[]> {
|
||||
): Promise<bigint[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const mapper = this.requireEntityIdMapper()
|
||||
const targetId = mapper.getUuid(Number(targetInt))
|
||||
if (targetId === undefined) return []
|
||||
|
||||
const startTime = performance.now()
|
||||
const verbIds = await this.lsmTreeVerbsByTarget.get(targetId)
|
||||
const elapsed = performance.now() - startTime
|
||||
|
|
@ -373,19 +448,42 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
prodLog.warn(`GraphAdjacencyIndex: Slow getVerbIdsByTarget for ${targetId}: ${elapsed.toFixed(2)}ms`)
|
||||
}
|
||||
|
||||
// Filter out deleted verb IDs (tombstone deletion workaround)
|
||||
// LSM-tree retains all IDs, but verbIdSet tracks deletions
|
||||
const allIds = verbIds || []
|
||||
return this.verbIdsToPaginatedInts(verbIds || [], options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared tail for the verb-int read methods: drop tombstoned ids (LSM trees
|
||||
* retain all ids; verbIdSet tracks deletions), apply pagination, and intern
|
||||
* the survivors to verb ints. Interning on the read path is safe — ids are
|
||||
* assigned deterministically within the process lifetime and never persisted.
|
||||
*/
|
||||
private verbIdsToPaginatedInts(
|
||||
allIds: string[],
|
||||
options?: { limit?: number; offset?: number }
|
||||
): bigint[] {
|
||||
let result = allIds.filter(id => this.verbIdSet.has(id))
|
||||
|
||||
// Apply pagination if requested
|
||||
if (options?.limit !== undefined || options?.offset !== undefined) {
|
||||
const offset = options.offset || 0
|
||||
const limit = options.limit !== undefined ? options.limit : result.length
|
||||
const offset = options?.offset || 0
|
||||
const limit = options?.limit !== undefined ? options.limit : result.length
|
||||
result = result.slice(offset, offset + limit)
|
||||
}
|
||||
|
||||
return result
|
||||
return result.map(id => BigInt(this.internVerbId(id)))
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Batch reverse resolver: verb ints → verb-id strings (the
|
||||
* REQUIRED half of the 8.0 contract Brainy's warm cache feeds from). Reads
|
||||
* the in-process interning map populated by `addVerb`, `rebuild()`, and the
|
||||
* verb-id-set recovery path — the JS index derives the interning from
|
||||
* storage on cold start, so no sidecar persistence exists or is needed.
|
||||
* @param verbInts - Verb ints as returned by the verb-int read methods.
|
||||
* @returns One entry per input, order-preserving; `null` for unknown ints.
|
||||
*/
|
||||
async verbIntsToIds(verbInts: bigint[]): Promise<(string | null)[]> {
|
||||
return verbInts.map(verbInt => this.verbIntToId[Number(verbInt)] ?? null)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -529,15 +627,32 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
}
|
||||
|
||||
/**
|
||||
* Add relationship to index using LSM-tree storage
|
||||
* @description Add a relationship to the index (BigInt boundary). The
|
||||
* coordinator resolves both endpoint ints via `idMapper.getOrAssign` and
|
||||
* mirrors them onto `verb.sourceInt`/`verb.targetInt` before calling. The
|
||||
* JS index keys its LSM trees by the verb's endpoint UUIDs, so the int
|
||||
* params carry no extra information here — they exist for contract parity
|
||||
* with native providers whose trees are int-keyed.
|
||||
* @param verb - The verb to index (endpoint UUIDs are authoritative).
|
||||
* @param sourceInt - The source entity's interned int (contract parity).
|
||||
* @param targetInt - The target entity's interned int (contract parity).
|
||||
* @returns The interned verb int for `verb.id` (stable for the index lifetime).
|
||||
*/
|
||||
async addVerb(verb: GraphVerb): Promise<void> {
|
||||
async addVerb(verb: GraphVerb, sourceInt: bigint, targetInt: bigint): Promise<bigint> {
|
||||
await this.ensureInitialized()
|
||||
return BigInt(await this.indexVerb(verb))
|
||||
}
|
||||
|
||||
/**
|
||||
* String-keyed indexing core shared by {@link addVerb} and {@link rebuild}.
|
||||
* Returns the interned verb int.
|
||||
*/
|
||||
private async indexVerb(verb: GraphVerb): Promise<number> {
|
||||
const startTime = performance.now()
|
||||
|
||||
// Track verb ID (memory-efficient: IDs only, full objects loaded on-demand via UnifiedCache)
|
||||
this.verbIdSet.add(verb.id)
|
||||
const verbInt = this.internVerbId(verb.id)
|
||||
|
||||
// Add to LSM-trees (outgoing and incoming edges)
|
||||
await this.lsmTreeSource.add(verb.sourceId, verb.targetId)
|
||||
|
|
@ -561,12 +676,17 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
if (elapsed > 10.0) {
|
||||
prodLog.warn(`GraphAdjacencyIndex: Slow addVerb for ${verb.id}: ${elapsed.toFixed(2)}ms`)
|
||||
}
|
||||
|
||||
return verbInt
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove relationship from index
|
||||
* Note: LSM-tree edges persist (tombstone deletion not yet implemented)
|
||||
* Only removes from verb cache and updates counts
|
||||
* @description Remove a relationship from the index by its id string.
|
||||
* LSM-tree edges persist (tombstone deletion via verbIdSet filtering); the
|
||||
* verb's interned int is intentionally retained so previously returned verb
|
||||
* ints stay resolvable via {@link verbIntsToIds}.
|
||||
* @param verbId - The verb's UUID string.
|
||||
* @returns Resolves once the verb no longer appears in reads.
|
||||
*/
|
||||
async removeVerb(verbId: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -624,6 +744,11 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
this.totalRelationshipsIndexed = 0
|
||||
// CRITICAL FIX - Clear relationship counts to prevent accumulation
|
||||
this.relationshipCountsByType.clear()
|
||||
// Re-derive verb-int interning from scratch — it's process-lifetime
|
||||
// derived state (never persisted), so a rebuild starts a fresh
|
||||
// generation of verb ints alongside the fresh verbIdSet.
|
||||
this.verbIdToInt.clear()
|
||||
this.verbIntToId = []
|
||||
|
||||
// Note: LSM-trees will be recreated from storage via their own initialization
|
||||
// Verb data will be loaded on-demand via UnifiedCache
|
||||
|
|
@ -655,7 +780,7 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
confidence: verb.confidence,
|
||||
weight: verb.weight
|
||||
}
|
||||
await this.addVerb(graphVerb)
|
||||
await this.indexVerb(graphVerb)
|
||||
totalVerbs++
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,12 +58,16 @@ interface HeapEntry {
|
|||
/**
|
||||
* Unified column store coordinator.
|
||||
*
|
||||
* Entity ints cross the {@link ColumnStoreProvider} boundary as `bigint`
|
||||
* (8.0 u64 contract); internally this JS baseline stays u32 — `Number(bigint)`
|
||||
* narrowing is lossless under the `EntityIdSpaceExceeded` u32 guard.
|
||||
*
|
||||
* @example
|
||||
* const store = new ColumnStore()
|
||||
* await store.init(storage, idMapper)
|
||||
* store.addEntity(42, { createdAt: Date.now(), status: 'active' })
|
||||
* store.addEntity(42n, { createdAt: Date.now(), status: 'active' })
|
||||
* await store.flush()
|
||||
* const newest = await store.sortTopK('createdAt', 'desc', 10)
|
||||
* const newest = await store.sortTopK('createdAt', 'desc', 10) // bigint[]
|
||||
*/
|
||||
export class ColumnStore implements ColumnStoreProvider {
|
||||
private storage!: StorageAdapter
|
||||
|
|
@ -170,8 +174,13 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
* one entry per element is added.
|
||||
*
|
||||
* Auto-flushes when any tail buffer exceeds its threshold.
|
||||
*
|
||||
* @param entityIntId - The entity's interned integer ID (u64-safe BigInt;
|
||||
* narrowed to u32 internally under the `EntityIdSpaceExceeded` guard)
|
||||
* @param fields - Map of field name → value (or array of values for multi-value)
|
||||
*/
|
||||
addEntity(entityIntId: number, fields: Record<string, unknown>): void {
|
||||
addEntity(entityIntId: bigint, fields: Record<string, unknown>): void {
|
||||
const intId = Number(entityIntId)
|
||||
for (const [field, rawValue] of Object.entries(fields)) {
|
||||
if (rawValue === undefined || rawValue === null) continue
|
||||
|
||||
|
|
@ -179,11 +188,11 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
if (Array.isArray(rawValue)) {
|
||||
for (const v of rawValue) {
|
||||
if (v !== undefined && v !== null) {
|
||||
this.pushToBuffer(field, v, entityIntId, true)
|
||||
this.pushToBuffer(field, v, intId, true)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
this.pushToBuffer(field, rawValue, entityIntId, false)
|
||||
this.pushToBuffer(field, rawValue, intId, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -195,11 +204,16 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
* during ALL queries) and removes any pending entries from tail buffers.
|
||||
* The global bitmap is persisted in the manifest on flush and cleared
|
||||
* during compaction.
|
||||
*
|
||||
* @param entityIntId - The entity's interned integer ID (u64-safe BigInt;
|
||||
* narrowed to u32 internally under the `EntityIdSpaceExceeded` guard)
|
||||
*/
|
||||
removeEntity(entityIntId: number): void {
|
||||
removeEntity(entityIntId: bigint): void {
|
||||
const intId = Number(entityIntId)
|
||||
|
||||
// Remove from tail buffers (pending writes)
|
||||
for (const [, buffer] of this.tailBuffers) {
|
||||
buffer.remove(entityIntId)
|
||||
buffer.remove(intId)
|
||||
}
|
||||
|
||||
// Add to global deleted bitmap for every known field
|
||||
|
|
@ -209,7 +223,7 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
deleted = new RoaringBitmap32()
|
||||
this.deletedEntities.set(field, deleted)
|
||||
}
|
||||
deleted.add(entityIntId)
|
||||
deleted.add(intId)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -273,26 +287,28 @@ export class ColumnStore implements ColumnStoreProvider {
|
|||
}
|
||||
|
||||
/**
|
||||
* Sort top-K: return K entity int IDs in sorted order.
|
||||
* Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt).
|
||||
*
|
||||
* Uses k-way merge across all segment cursors + tail buffer via a min/max heap.
|
||||
* Complexity: O(K log S) where S = number of cursors.
|
||||
*/
|
||||
async sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise<number[]> {
|
||||
return this.mergeSort(field, order, k, null)
|
||||
async sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise<bigint[]> {
|
||||
const intIds = await this.mergeSort(field, order, k, null)
|
||||
return intIds.map((id) => BigInt(id))
|
||||
}
|
||||
|
||||
/**
|
||||
* Filtered sort top-K: return K entity int IDs in sorted order,
|
||||
* considering only entities present in the filter bitmap.
|
||||
* Filtered sort top-K: return K entity int IDs in sorted order (u64-safe
|
||||
* BigInt), considering only entities present in the filter bitmap.
|
||||
*/
|
||||
async filteredSortTopK(
|
||||
filterBitmap: RoaringBitmap32,
|
||||
field: string,
|
||||
order: 'asc' | 'desc',
|
||||
k: number
|
||||
): Promise<number[]> {
|
||||
return this.mergeSort(field, order, k, filterBitmap)
|
||||
): Promise<bigint[]> {
|
||||
const intIds = await this.mergeSort(field, order, k, filterBitmap)
|
||||
return intIds.map((id) => BigInt(id))
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -170,13 +170,19 @@ export const FLAG_MULTI_VALUE = 0x01
|
|||
* Brainy ships a TypeScript baseline that implements this interface.
|
||||
* Cortex registers a Rust-accelerated implementation at higher priority.
|
||||
* The MetadataIndex coordinator calls whichever is registered.
|
||||
*
|
||||
* **8.0 u64 contract — BigInt entity ints at the boundary.** Entity ints flow
|
||||
* in and out as `bigint` so a native u64-keyed column store loses nothing at
|
||||
* the wrapper. Brainy's JS baseline stays u32 internally (`Number(bigint)`
|
||||
* narrowing is lossless under the shipped `EntityIdSpaceExceeded` u32 guard)
|
||||
* and its filter/range bitmaps stay Roaring32 — only this boundary widens.
|
||||
*/
|
||||
export interface ColumnStoreProvider {
|
||||
/**
|
||||
* Initialize the column store: discover fields, load manifests.
|
||||
*
|
||||
* @param storage - The storage adapter for reading/writing segment files
|
||||
* @param idMapper - Shared EntityIdMapper for UUID ↔ u32 conversion
|
||||
* @param idMapper - Shared EntityIdMapper for UUID ↔ int conversion
|
||||
*/
|
||||
init(storage: StorageAdapter, idMapper: EntityIdMapper): Promise<void>
|
||||
|
||||
|
|
@ -184,17 +190,17 @@ export interface ColumnStoreProvider {
|
|||
* Index an entity's field values. For multi-value fields (arrays),
|
||||
* one entry per element is added to the column.
|
||||
*
|
||||
* @param entityIntId - The entity's u32 integer ID (from EntityIdMapper)
|
||||
* @param entityIntId - The entity's interned integer ID (u64-safe BigInt)
|
||||
* @param fields - Map of field name → value (or array of values for multi-value)
|
||||
*/
|
||||
addEntity(entityIntId: number, fields: Record<string, unknown>): void
|
||||
addEntity(entityIntId: bigint, fields: Record<string, unknown>): void
|
||||
|
||||
/**
|
||||
* Remove an entity from all indexed columns by adding tombstones.
|
||||
*
|
||||
* @param entityIntId - The entity's u32 integer ID
|
||||
* @param entityIntId - The entity's interned integer ID (u64-safe BigInt)
|
||||
*/
|
||||
removeEntity(entityIntId: number): void
|
||||
removeEntity(entityIntId: bigint): void
|
||||
|
||||
/**
|
||||
* Point filter: find all entities where field equals value.
|
||||
|
|
@ -222,9 +228,9 @@ export interface ColumnStoreProvider {
|
|||
* @param field - Field to sort by
|
||||
* @param order - 'asc' or 'desc'
|
||||
* @param k - Maximum number of results
|
||||
* @returns Sorted array of entity int IDs
|
||||
* @returns Sorted array of entity int IDs (u64-safe BigInt)
|
||||
*/
|
||||
sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise<number[]>
|
||||
sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise<bigint[]>
|
||||
|
||||
/**
|
||||
* Filtered sort top-K: same as sortTopK but only considers entities
|
||||
|
|
@ -234,14 +240,14 @@ export interface ColumnStoreProvider {
|
|||
* @param field - Field to sort by
|
||||
* @param order - 'asc' or 'desc'
|
||||
* @param k - Maximum number of results
|
||||
* @returns Sorted array of entity int IDs (subset of filterBitmap)
|
||||
* @returns Sorted array of entity int IDs (u64-safe BigInt, subset of filterBitmap)
|
||||
*/
|
||||
filteredSortTopK(
|
||||
filterBitmap: RoaringBitmap32,
|
||||
field: string,
|
||||
order: 'asc' | 'desc',
|
||||
k: number
|
||||
): Promise<number[]>
|
||||
): Promise<bigint[]>
|
||||
|
||||
/**
|
||||
* Get all distinct values for a field across all segments.
|
||||
|
|
|
|||
|
|
@ -150,8 +150,20 @@ export interface MetadataIndexProvider {
|
|||
|
||||
getStats(): Promise<MetadataIndexStats>
|
||||
|
||||
/** The shared UUID ↔ int mapper. Brainy reads `.getUuid(intId)` off the result. */
|
||||
getIdMapper(): { getUuid(intId: number): string | undefined }
|
||||
/**
|
||||
* The shared UUID ↔ int mapper — the single source of truth for entity-int
|
||||
* resolution at the provider boundary. The coordinator (`brainy.ts`) resolves
|
||||
* UUID → int exactly once before every graph-index call (`getOrAssign` on
|
||||
* writes, `getInt` on reads — `undefined` means "never mapped", i.e. the
|
||||
* entity has no relations) and converts provider-returned ints back with
|
||||
* `getUuid`. Ints are u32 today (the `EntityIdSpaceExceeded` guard enforces
|
||||
* the ceiling on the JS path), so `Number(bigint)` narrowing is lossless.
|
||||
*/
|
||||
getIdMapper(): {
|
||||
getOrAssign(uuid: string): number
|
||||
getInt(uuid: string): number | undefined
|
||||
getUuid(intId: number): string | undefined
|
||||
}
|
||||
|
||||
/** The column store the coordinator delegates `where`/`orderBy` to. */
|
||||
readonly columnStore: import('./indexes/columnStore/types.js').ColumnStoreProvider
|
||||
|
|
@ -160,19 +172,75 @@ export interface MetadataIndexProvider {
|
|||
/**
|
||||
* The `'graphIndex'` provider — a drop-in for `GraphAdjacencyIndex`.
|
||||
* Brainy calls this surface via `this.graphIndex.*` (some optional-chained).
|
||||
*
|
||||
* **8.0 u64 contract — BigInt at the boundary.** Reads take entity ints
|
||||
* (from the metadata index's idMapper) and return entity/verb ints as
|
||||
* `bigint[]`. The coordinator owns ALL UUID ↔ int conversion: it resolves
|
||||
* UUIDs to ints once at the `brainy.ts` boundary (`getOrAssign` on writes,
|
||||
* `getInt` on reads, returning empty results for unmapped UUIDs without
|
||||
* calling the provider) and maps returned ints back (`getUuid` for entities,
|
||||
* {@link GraphIndexProvider.verbIntsToIds} for verbs). Implementations may
|
||||
* stay u32 internally — `Number(bigint)` narrowing is lossless under the
|
||||
* shipped `EntityIdSpaceExceeded` u32 guard — but must speak the bigint
|
||||
* contract at this surface.
|
||||
*/
|
||||
export interface GraphIndexProvider {
|
||||
/** `false` until the index has loaded; Brainy probes this before fast paths. */
|
||||
readonly isInitialized: boolean
|
||||
|
||||
/**
|
||||
* @description Entity ints reachable from `id` (1 hop), deduped.
|
||||
* @param id - The entity's interned int (from the shared idMapper).
|
||||
* @param options - Direction (`'both'` default) and limit/offset pagination.
|
||||
* @returns Neighbor entity ints. Empty when the entity has no edges.
|
||||
*/
|
||||
getNeighbors(
|
||||
id: string,
|
||||
optionsOrDirection?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number } | 'in' | 'out' | 'both'
|
||||
): Promise<string[]>
|
||||
getVerbIdsBySource(sourceId: string, options?: { limit?: number; offset?: number }): Promise<string[]>
|
||||
getVerbIdsByTarget(targetId: string, options?: { limit?: number; offset?: number }): Promise<string[]>
|
||||
id: bigint,
|
||||
options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number }
|
||||
): Promise<bigint[]>
|
||||
/**
|
||||
* @description Verb ints for all edges originating at `sourceInt`.
|
||||
* @param sourceInt - The source entity's interned int.
|
||||
* @param options - Optional limit/offset pagination.
|
||||
* @returns Verb ints, resolvable via {@link GraphIndexProvider.verbIntsToIds}.
|
||||
*/
|
||||
getVerbIdsBySource(sourceInt: bigint, options?: { limit?: number; offset?: number }): Promise<bigint[]>
|
||||
/**
|
||||
* @description Verb ints for all edges pointing at `targetInt`.
|
||||
* @param targetInt - The target entity's interned int.
|
||||
* @param options - Optional limit/offset pagination.
|
||||
* @returns Verb ints, resolvable via {@link GraphIndexProvider.verbIntsToIds}.
|
||||
*/
|
||||
getVerbIdsByTarget(targetInt: bigint, options?: { limit?: number; offset?: number }): Promise<bigint[]>
|
||||
/**
|
||||
* @description Batch reverse resolver: verb ints → verb-id strings. REQUIRED —
|
||||
* the provider owns the durable verb-int interning (Brainy keeps only a
|
||||
* bounded in-memory warm cache fed by `addVerb` returns and this resolver;
|
||||
* pure optimization, no durability role).
|
||||
* @param verbInts - Verb ints as returned by the read methods.
|
||||
* @returns One entry per input, order-preserving; `null` for unknown ints.
|
||||
*/
|
||||
verbIntsToIds(verbInts: bigint[]): Promise<(string | null)[]>
|
||||
getVerbsBatchCached(verbIds: string[]): Promise<Map<string, GraphVerb>>
|
||||
|
||||
/**
|
||||
* @description Index one verb. The coordinator resolves both endpoint ints
|
||||
* via `idMapper.getOrAssign` and mirrors them onto `verb.sourceInt` /
|
||||
* `verb.targetInt` before the call.
|
||||
* @param verb - The verb to index (endpoint UUIDs + derived `sourceInt`/`targetInt`).
|
||||
* @param sourceInt - The source entity's interned int.
|
||||
* @param targetInt - The target entity's interned int.
|
||||
* @returns The interned verb int for `verb.id` (feeds Brainy's warm cache).
|
||||
*/
|
||||
addVerb(verb: GraphVerb, sourceInt: bigint, targetInt: bigint): Promise<bigint>
|
||||
/**
|
||||
* @description Remove one verb from the index by its id string. The verb's
|
||||
* interned int stays reserved (ints are never recycled within a generation).
|
||||
* @param verbId - The verb's UUID string.
|
||||
* @returns Resolves once the verb no longer appears in reads.
|
||||
*/
|
||||
removeVerb(verbId: string): Promise<void>
|
||||
|
||||
rebuild(): Promise<void>
|
||||
flush(): Promise<void>
|
||||
close(): Promise<void>
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
*/
|
||||
|
||||
import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js'
|
||||
import type { GraphEntityIdResolver } from '../graph/graphAdjacencyIndex.js'
|
||||
|
||||
import {
|
||||
GraphVerb,
|
||||
|
|
@ -176,6 +177,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
protected isInitialized = false
|
||||
protected graphIndex?: GraphAdjacencyIndex
|
||||
protected graphIndexPromise?: Promise<GraphAdjacencyIndex>
|
||||
/**
|
||||
* Shared UUID ↔ int resolver for the graph index's BigInt boundary.
|
||||
* Wired by Brainy via {@link setGraphEntityIdResolver}; until then the verb
|
||||
* read paths fall back to shard iteration.
|
||||
*/
|
||||
protected graphEntityIdResolver?: GraphEntityIdResolver
|
||||
protected readOnly = false
|
||||
|
||||
// Write-through cache for read-after-write consistency
|
||||
|
|
@ -442,6 +449,26 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
this.graphIndexPromise = Promise.resolve(index)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Wire the shared UUID ↔ int resolver used at the graph index's
|
||||
* BigInt boundary (8.0 u64 contract). Brainy calls this with
|
||||
* `metadataIndex.getIdMapper()` after the graph index is resolved (init,
|
||||
* fork, and checkout paths). The storage layer needs it to convert UUIDs to
|
||||
* entity ints before `getVerbIdsBySource`/`getVerbIdsByTarget` calls and to
|
||||
* resolve returned verb ints back to verb-id strings. Until it's wired, the
|
||||
* verb read paths fall back to shard iteration (correct, just slower).
|
||||
* @param resolver - The shared entity-id resolver.
|
||||
* @returns Nothing.
|
||||
*/
|
||||
public setGraphEntityIdResolver(resolver: GraphEntityIdResolver): void {
|
||||
this.graphEntityIdResolver = resolver
|
||||
// Thread the resolver into the JS graph index too, when present — native
|
||||
// providers carry their own mapper and don't expose this setter.
|
||||
if (this.graphIndex && typeof (this.graphIndex as GraphAdjacencyIndex).setEntityIdMapper === 'function') {
|
||||
(this.graphIndex as GraphAdjacencyIndex).setEntityIdMapper(resolver)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the storage adapter is initialized
|
||||
*/
|
||||
|
|
@ -2107,7 +2134,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
*/
|
||||
private async _initializeGraphIndex(): Promise<GraphAdjacencyIndex> {
|
||||
prodLog.info('Initializing GraphAdjacencyIndex...')
|
||||
this.graphIndex = new GraphAdjacencyIndex(this)
|
||||
// Thread the shared entity-id resolver when already wired (re-init after
|
||||
// invalidateGraphIndex); on first init Brainy wires it right after.
|
||||
this.graphIndex = new GraphAdjacencyIndex(this, {}, this.graphEntityIdResolver)
|
||||
|
||||
// Check if we need to rebuild from existing data
|
||||
const sampleVerbs = await this.getVerbs({ pagination: { limit: 1 } })
|
||||
|
|
@ -3610,10 +3639,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`)
|
||||
|
||||
// Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
|
||||
if (this.graphIndex && this.graphIndex.isInitialized) {
|
||||
// Fast path - use GraphAdjacencyIndex if available (lazy-loaded).
|
||||
// 8.0 BigInt boundary: convert the UUID to an entity int up front and
|
||||
// resolve returned verb ints back to verb-id strings.
|
||||
if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) {
|
||||
try {
|
||||
const verbIds = await this.graphIndex.getVerbIdsBySource(sourceId)
|
||||
const sourceInt = this.graphEntityIdResolver.getInt(sourceId)
|
||||
if (sourceInt === undefined) {
|
||||
// Never-mapped UUID — the entity has no relations by definition.
|
||||
return []
|
||||
}
|
||||
const verbInts = await this.graphIndex.getVerbIdsBySource(BigInt(sourceInt))
|
||||
const verbIds = (await this.graphIndex.verbIntsToIds(verbInts))
|
||||
.filter((id): id is string => id !== null)
|
||||
prodLog.debug(`[BaseStorage] GraphAdjacencyIndex found ${verbIds.length} verb IDs for sourceId=${sourceId}`)
|
||||
|
||||
// PERFORMANCE FIX - Batch fetch verbs + metadata (eliminates N+1 pattern)
|
||||
|
|
@ -3860,10 +3898,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
): Promise<HNSWVerbWithMetadata[]> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Fast path - use GraphAdjacencyIndex if available (lazy-loaded)
|
||||
if (this.graphIndex && this.graphIndex.isInitialized) {
|
||||
// Fast path - use GraphAdjacencyIndex if available (lazy-loaded).
|
||||
// 8.0 BigInt boundary: convert the UUID to an entity int up front and
|
||||
// resolve returned verb ints back to verb-id strings.
|
||||
if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) {
|
||||
try {
|
||||
const verbIds = await this.graphIndex.getVerbIdsByTarget(targetId)
|
||||
const targetInt = this.graphEntityIdResolver.getInt(targetId)
|
||||
if (targetInt === undefined) {
|
||||
// Never-mapped UUID — the entity has no relations by definition.
|
||||
return []
|
||||
}
|
||||
const verbInts = await this.graphIndex.getVerbIdsByTarget(BigInt(targetInt))
|
||||
const verbIds = (await this.graphIndex.verbIntsToIds(verbInts))
|
||||
.filter((id): id is string => id !== null)
|
||||
const results: HNSWVerbWithMetadata[] = []
|
||||
|
||||
for (const verbId of verbIds) {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
|
||||
import type { JsHnswVectorIndex } from '../../hnsw/hnswIndex.js'
|
||||
import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
|
||||
import type { GraphAdjacencyIndex } from '../../graph/graphAdjacencyIndex.js'
|
||||
import type { GraphIndexProvider } from '../../plugin.js'
|
||||
import type { GraphVerb } from '../../coreTypes.js'
|
||||
import type { Operation, RollbackAction } from '../types.js'
|
||||
|
||||
|
|
@ -152,18 +152,35 @@ export class RemoveFromMetadataIndexOperation implements Operation {
|
|||
*
|
||||
* Rollback strategy:
|
||||
* - Remove verb from graph index
|
||||
*
|
||||
* 8.0 u64 contract: the coordinator resolves both endpoint ints via the
|
||||
* shared idMapper (`getOrAssign`) and passes them alongside the verb; the
|
||||
* provider returns the interned verb int, which is surfaced through the
|
||||
* optional `onVerbInt` callback so the coordinator can feed its warm cache.
|
||||
*/
|
||||
export class AddToGraphIndexOperation implements Operation {
|
||||
readonly name = 'AddToGraphIndex'
|
||||
|
||||
/**
|
||||
* @param index - The graph-index provider (JS baseline or native).
|
||||
* @param verb - The verb to index (`sourceInt`/`targetInt` mirrored on it).
|
||||
* @param sourceInt - The source entity's interned int.
|
||||
* @param targetInt - The target entity's interned int.
|
||||
* @param onVerbInt - Optional hook invoked with the interned verb int
|
||||
* returned by the provider (feeds the coordinator's verb-int warm cache).
|
||||
*/
|
||||
constructor(
|
||||
private readonly index: GraphAdjacencyIndex,
|
||||
private readonly verb: GraphVerb
|
||||
private readonly index: GraphIndexProvider,
|
||||
private readonly verb: GraphVerb,
|
||||
private readonly sourceInt: bigint,
|
||||
private readonly targetInt: bigint,
|
||||
private readonly onVerbInt?: (verbInt: bigint) => void
|
||||
) {}
|
||||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
// Add verb to graph index
|
||||
await this.index.addVerb(this.verb)
|
||||
const verbInt = await this.index.addVerb(this.verb, this.sourceInt, this.targetInt)
|
||||
this.onVerbInt?.(verbInt)
|
||||
|
||||
// Return rollback action
|
||||
return async () => {
|
||||
|
|
@ -178,13 +195,25 @@ export class AddToGraphIndexOperation implements Operation {
|
|||
*
|
||||
* Rollback strategy:
|
||||
* - Re-add verb to graph index
|
||||
*
|
||||
* 8.0 u64 contract: rollback re-adds through `addVerb(verb, sourceInt,
|
||||
* targetInt)`, so the coordinator resolves the endpoint ints up front
|
||||
* (while the entity → int mappings are guaranteed to still exist).
|
||||
*/
|
||||
export class RemoveFromGraphIndexOperation implements Operation {
|
||||
readonly name = 'RemoveFromGraphIndex'
|
||||
|
||||
/**
|
||||
* @param index - The graph-index provider (JS baseline or native).
|
||||
* @param verb - The verb being removed (required for rollback re-add).
|
||||
* @param sourceInt - The source entity's interned int (rollback re-add).
|
||||
* @param targetInt - The target entity's interned int (rollback re-add).
|
||||
*/
|
||||
constructor(
|
||||
private readonly index: GraphAdjacencyIndex,
|
||||
private readonly verb: GraphVerb // Required for rollback
|
||||
private readonly index: GraphIndexProvider,
|
||||
private readonly verb: GraphVerb, // Required for rollback
|
||||
private readonly sourceInt: bigint,
|
||||
private readonly targetInt: bigint
|
||||
) {}
|
||||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
|
|
@ -194,7 +223,7 @@ export class RemoveFromGraphIndexOperation implements Operation {
|
|||
// Return rollback action
|
||||
return async () => {
|
||||
// Re-add verb with original data
|
||||
await this.index.addVerb(this.verb)
|
||||
await this.index.addVerb(this.verb, this.sourceInt, this.targetInt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -368,7 +368,7 @@ export interface GraphNoun {
|
|||
* Represents relationships between nouns
|
||||
*/
|
||||
export interface GraphVerb {
|
||||
id: string // Unique identifier for the verb
|
||||
id: string // Unique identifier — always a UUID (8.0 contract: brainy generates every verb id, so providers may key verb-int interning on the raw UUID bytes)
|
||||
sourceId: string // Entity UUID of the source noun
|
||||
targetId: string // Entity UUID of the target noun
|
||||
label?: string // Optional descriptive label
|
||||
|
|
@ -381,6 +381,21 @@ export interface GraphVerb {
|
|||
embedding?: number[] // Vector representation of the relationship
|
||||
confidence?: number // Confidence score (0-1)
|
||||
weight?: number // Strength/importance of the relationship
|
||||
|
||||
/**
|
||||
* Derived state: the source entity's interned integer ID (u64-safe BigInt).
|
||||
* Populated by the coordinator from the shared entity-id mapper at add time,
|
||||
* immediately before the verb is handed to the graph-index provider. NEVER
|
||||
* persisted to storage JSON — the mapper is the source of truth, and
|
||||
* persisting would denormalize state that can go stale across a mapper rebuild.
|
||||
*/
|
||||
sourceInt?: bigint
|
||||
/**
|
||||
* Derived state: the target entity's interned integer ID (u64-safe BigInt).
|
||||
* Same lifecycle as {@link GraphVerb.sourceInt}: coordinator-populated at add
|
||||
* time, never persisted to storage JSON.
|
||||
*/
|
||||
targetInt?: bigint
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1412,7 +1412,7 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
fieldsMap[field] = value
|
||||
}
|
||||
}
|
||||
this.columnStore.addEntity(entityIntId, fieldsMap)
|
||||
this.columnStore.addEntity(BigInt(entityIntId), fieldsMap)
|
||||
}
|
||||
|
||||
// Adaptive auto-flush based on usage patterns
|
||||
|
|
@ -1501,7 +1501,7 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
if (this.columnStore) {
|
||||
const intId = this.idMapper.getInt(id)
|
||||
if (intId !== undefined) {
|
||||
this.columnStore.removeEntity(intId)
|
||||
this.columnStore.removeEntity(BigInt(intId))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2073,7 +2073,7 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
|
||||
if (hasFilter && filteredIds.length === 0) return []
|
||||
|
||||
let sortedIntIds: number[]
|
||||
let sortedIntIds: bigint[]
|
||||
if (hasFilter) {
|
||||
// Build filter bitmap for the column store
|
||||
const filterBitmap = new RoaringBitmap32()
|
||||
|
|
@ -2091,9 +2091,10 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
)
|
||||
}
|
||||
|
||||
// Convert int IDs back to UUIDs
|
||||
// Convert int IDs back to UUIDs. Number() narrowing is lossless — the
|
||||
// shipped EntityIdSpaceExceeded guard caps the JS mapper at u32.
|
||||
return sortedIntIds
|
||||
.map(intId => this.idMapper.getUuid(intId))
|
||||
.map(intId => this.idMapper.getUuid(Number(intId)))
|
||||
.filter((uuid): uuid is string => uuid !== undefined)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -524,6 +524,21 @@ export function validateUpdateParams(params: UpdateParams): void {
|
|||
* Validate relate parameters
|
||||
*/
|
||||
export function validateRelateParams(params: RelateParams): void {
|
||||
// 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy.
|
||||
// RelateParams has no `id` field — an untyped caller passing one would
|
||||
// previously have it silently ignored (a generated UUID was used instead).
|
||||
// Teach instead of surprise: explain the contract and how ids actually work.
|
||||
const suppliedId = (params as RelateParams & { id?: unknown }).id
|
||||
if (suppliedId !== undefined) {
|
||||
throw new Error(
|
||||
`relate() does not accept a custom id (got: ${JSON.stringify(suppliedId)}). ` +
|
||||
'Verb ids are UUIDs by contract in 8.0 — brainy generates one for every ' +
|
||||
'relationship and returns it from relate(). Graph-index providers key ' +
|
||||
'verb-int interning on the raw UUID bytes, so custom verb ids are not ' +
|
||||
'supported. Remove the id field and use the returned id instead.'
|
||||
)
|
||||
}
|
||||
|
||||
// Universal truths
|
||||
if (!params.from) {
|
||||
throw new Error('from entity ID is required')
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js'
|
||||
import { EntityIdMapper } from '../../src/utils/entityIdMapper.js'
|
||||
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
|
||||
import { performance } from 'perf_hooks'
|
||||
|
||||
|
|
@ -112,8 +113,18 @@ describe('🧠 Graph Scale Performance Benchmarks', () => {
|
|||
let brain: Brainy
|
||||
let graphIndex: GraphAdjacencyIndex
|
||||
let storage: MemoryStorage
|
||||
let idMapper: EntityIdMapper
|
||||
const scale = getTestScale()
|
||||
|
||||
/** Resolve a UUID to its entity int for the 8.0 BigInt boundary. */
|
||||
const entityInt = (uuid: string): bigint => BigInt(idMapper.getOrAssign(uuid))
|
||||
|
||||
/** Map returned entity ints back to UUIDs. */
|
||||
const intsToUuids = (ints: bigint[]): string[] =>
|
||||
ints
|
||||
.map((i) => idMapper.getUuid(Number(i)))
|
||||
.filter((u): u is string => u !== undefined)
|
||||
|
||||
// Performance tracking
|
||||
const lookupStats = new PerformanceStats()
|
||||
const updateStats = new PerformanceStats()
|
||||
|
|
@ -130,10 +141,13 @@ describe('🧠 Graph Scale Performance Benchmarks', () => {
|
|||
storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
|
||||
idMapper = new EntityIdMapper({ storage })
|
||||
await idMapper.init()
|
||||
|
||||
graphIndex = new GraphAdjacencyIndex(storage, {
|
||||
maxIndexSize: scale.nodes,
|
||||
autoOptimize: true
|
||||
})
|
||||
}, idMapper)
|
||||
|
||||
// Initialize Brainy for unified testing
|
||||
brain = new Brainy({ requireSubtype: false,
|
||||
|
|
@ -226,7 +240,7 @@ describe('🧠 Graph Scale Performance Benchmarks', () => {
|
|||
|
||||
// Warm up the index
|
||||
await graphIndex.rebuild()
|
||||
await graphIndex.getNeighbors('node-100') // Warm up
|
||||
await graphIndex.getNeighbors(entityInt('node-100')) // Warm up
|
||||
|
||||
// Test random lookups
|
||||
const testIterations = Math.min(1000, scale.nodes / 10)
|
||||
|
|
@ -236,7 +250,7 @@ describe('🧠 Graph Scale Performance Benchmarks', () => {
|
|||
|
||||
for (const nodeId of sampleNodes) {
|
||||
const startTime = performance.now()
|
||||
const neighbors = await graphIndex.getNeighbors(nodeId)
|
||||
const neighbors = await graphIndex.getNeighbors(entityInt(nodeId))
|
||||
const elapsed = performance.now() - startTime
|
||||
|
||||
lookupStats.addSample(elapsed)
|
||||
|
|
@ -408,9 +422,9 @@ describe('🧠 Graph Scale Performance Benchmarks', () => {
|
|||
visited.add(id)
|
||||
nodesTraversed++
|
||||
|
||||
// Get neighbors
|
||||
const neighbors = await graphIndex.getNeighbors(id, 'out')
|
||||
for (const neighbor of neighbors) {
|
||||
// Get neighbors (BigInt boundary: ints out, mapped back to UUIDs)
|
||||
const neighborInts = await graphIndex.getNeighbors(entityInt(id), { direction: 'out' })
|
||||
for (const neighbor of intsToUuids(neighborInts)) {
|
||||
if (!visited.has(neighbor)) {
|
||||
queue.push({ id: neighbor, depth: depth + 1 })
|
||||
}
|
||||
|
|
@ -687,7 +701,7 @@ describe('🧠 Graph Scale Performance Benchmarks', () => {
|
|||
|
||||
switch (operationType) {
|
||||
case 0: // Neighbor lookup
|
||||
operations.push(graphIndex.getNeighbors(`node-${Math.floor(Math.random() * scale.nodes)}`))
|
||||
operations.push(graphIndex.getNeighbors(entityInt(`node-${Math.floor(Math.random() * scale.nodes)}`)))
|
||||
break
|
||||
case 1: // Unified find
|
||||
operations.push(brain.find({
|
||||
|
|
|
|||
|
|
@ -151,8 +151,12 @@ describe('Duplicate Check Optimization', () => {
|
|||
type: VerbType.RelatesTo
|
||||
})
|
||||
|
||||
// Access GraphIndex to ensure verb is cached
|
||||
const verbIds = await (brain as any).graphIndex.getVerbIdsBySource(entityA)
|
||||
// Access GraphIndex to ensure verb is cached (8.0 BigInt boundary:
|
||||
// UUID → entity int in, verb ints out, resolved back via verbIntsToIds)
|
||||
const idMapper = (brain as any).metadataIndex.getIdMapper()
|
||||
const sourceInt = BigInt(idMapper.getInt(entityA)!)
|
||||
const verbInts: bigint[] = await (brain as any).graphIndex.getVerbIdsBySource(sourceInt)
|
||||
const verbIds = await (brain as any).graphIndex.verbIntsToIds(verbInts)
|
||||
expect(verbIds).toContain(relationId1)
|
||||
|
||||
// Attempt duplicate (should use cached verb)
|
||||
|
|
|
|||
230
tests/unit/graph/bigint-contract.test.ts
Normal file
230
tests/unit/graph/bigint-contract.test.ts
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
/**
|
||||
* @module bigint-contract.test
|
||||
* @description 8.0 u64 provider contract — BigInt boundary of the JS
|
||||
* GraphAdjacencyIndex and the coordinator that drives it.
|
||||
*
|
||||
* Covers:
|
||||
* - bigint round-trip through the JS graph index: `addVerb` returns a stable
|
||||
* verb int, `getVerbIdsBySource`/`getVerbIdsByTarget` return that int, and
|
||||
* `verbIntsToIds` maps it back to the verb-id string (null for unknown ints)
|
||||
* - `getNeighbors` speaks entity ints both ways via the shared idMapper
|
||||
* - unmapped entity ints / UUIDs produce empty reads, never errors
|
||||
* - a missing idMapper fails loudly (wiring bug, not a silent fallback)
|
||||
* - coordinator-level behavior: relate → getRelations/neighbors → unrelate
|
||||
* works end-to-end over the BigInt boundary (public API unchanged)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { Brainy } from '../../../src/brainy.js'
|
||||
import { GraphAdjacencyIndex } from '../../../src/graph/graphAdjacencyIndex.js'
|
||||
import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import { NounType, VerbType } from '../../../src/types/graphTypes.js'
|
||||
import type { GraphVerb } from '../../../src/coreTypes.js'
|
||||
|
||||
// Verb ids are UUID-shaped by contract in 8.0 (storage enforces the shape on
|
||||
// every save; cortex keys verb-int interning on the raw UUID bytes).
|
||||
const VERB_UUID_1 = '11111111-1111-4111-8111-111111111111'
|
||||
const VERB_UUID_2 = '22222222-2222-4222-8222-222222222222'
|
||||
const VERB_UUID_3 = '33333333-3333-4333-8333-333333333333'
|
||||
|
||||
/** Build a minimal GraphVerb for direct index-level tests. */
|
||||
function makeVerb(id: string, sourceId: string, targetId: string): GraphVerb {
|
||||
return {
|
||||
id,
|
||||
sourceId,
|
||||
targetId,
|
||||
vector: [],
|
||||
type: VerbType.RelatedTo,
|
||||
verb: VerbType.RelatedTo
|
||||
}
|
||||
}
|
||||
|
||||
describe('GraphAdjacencyIndex — BigInt boundary (JS implementation)', () => {
|
||||
let storage: MemoryStorage
|
||||
let idMapper: EntityIdMapper
|
||||
let index: GraphAdjacencyIndex
|
||||
|
||||
beforeEach(async () => {
|
||||
storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
idMapper = new EntityIdMapper({ storage, storageKey: 'test:graph:idMapper' })
|
||||
await idMapper.init()
|
||||
index = new GraphAdjacencyIndex(storage, {}, idMapper)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await index.close()
|
||||
})
|
||||
|
||||
it('addVerb returns a stable verb int; reads return it; verbIntsToIds maps back', async () => {
|
||||
const sourceInt = BigInt(idMapper.getOrAssign('uuid-source'))
|
||||
const targetInt = BigInt(idMapper.getOrAssign('uuid-target'))
|
||||
const verb = makeVerb(VERB_UUID_1, 'uuid-source', 'uuid-target')
|
||||
verb.sourceInt = sourceInt
|
||||
verb.targetInt = targetInt
|
||||
|
||||
const verbInt = await index.addVerb(verb, sourceInt, targetInt)
|
||||
expect(typeof verbInt).toBe('bigint')
|
||||
|
||||
// Re-adding the same verb id returns the SAME interned int (stable).
|
||||
const verbIntAgain = await index.addVerb(verb, sourceInt, targetInt)
|
||||
expect(verbIntAgain).toBe(verbInt)
|
||||
|
||||
// Both directed verb-int reads surface the int from addVerb.
|
||||
const bySource = await index.getVerbIdsBySource(sourceInt)
|
||||
expect(bySource).toContain(verbInt)
|
||||
|
||||
const byTarget = await index.getVerbIdsByTarget(targetInt)
|
||||
expect(byTarget).toContain(verbInt)
|
||||
|
||||
// Batch reverse resolution is order-preserving with null for unknowns.
|
||||
const resolved = await index.verbIntsToIds([verbInt, 999_999n])
|
||||
expect(resolved).toEqual([VERB_UUID_1, null])
|
||||
})
|
||||
|
||||
it('getNeighbors speaks entity ints in both directions', async () => {
|
||||
const aInt = BigInt(idMapper.getOrAssign('uuid-a'))
|
||||
const bInt = BigInt(idMapper.getOrAssign('uuid-b'))
|
||||
const verb = makeVerb(VERB_UUID_2, 'uuid-a', 'uuid-b')
|
||||
|
||||
await index.addVerb(verb, aInt, bInt)
|
||||
|
||||
const out = await index.getNeighbors(aInt, { direction: 'out' })
|
||||
expect(out).toEqual([bInt])
|
||||
|
||||
const incoming = await index.getNeighbors(bInt, { direction: 'in' })
|
||||
expect(incoming).toEqual([aInt])
|
||||
|
||||
const both = await index.getNeighbors(aInt)
|
||||
expect(both).toContain(bInt)
|
||||
})
|
||||
|
||||
it('reads for unmapped entity ints return empty without error', async () => {
|
||||
// 424242 was never assigned by the mapper — no UUID behind it.
|
||||
const ghost = 424_242n
|
||||
expect(await index.getNeighbors(ghost)).toEqual([])
|
||||
expect(await index.getVerbIdsBySource(ghost)).toEqual([])
|
||||
expect(await index.getVerbIdsByTarget(ghost)).toEqual([])
|
||||
})
|
||||
|
||||
it('verb ints stay resolvable after removeVerb (interning is append-only)', async () => {
|
||||
const sInt = BigInt(idMapper.getOrAssign('uuid-s'))
|
||||
const tInt = BigInt(idMapper.getOrAssign('uuid-t'))
|
||||
const verb = makeVerb(VERB_UUID_3, 'uuid-s', 'uuid-t')
|
||||
|
||||
const verbInt = await index.addVerb(verb, sInt, tInt)
|
||||
// Persist the verb (vector + metadata — getVerb requires both) so
|
||||
// removeVerb can resolve its type for count updates.
|
||||
await storage.saveVerb({
|
||||
id: verb.id,
|
||||
vector: [],
|
||||
connections: new Map<number, Set<string>>(),
|
||||
verb: VerbType.RelatedTo,
|
||||
sourceId: verb.sourceId,
|
||||
targetId: verb.targetId
|
||||
})
|
||||
await storage.saveVerbMetadata(verb.id, {
|
||||
verb: VerbType.RelatedTo,
|
||||
createdAt: Date.now()
|
||||
})
|
||||
|
||||
await index.removeVerb(verb.id)
|
||||
|
||||
// Removed verb no longer appears in reads…
|
||||
expect(await index.getVerbIdsBySource(sInt)).toEqual([])
|
||||
// …but its int still reverse-resolves (append-only interning).
|
||||
expect(await index.verbIntsToIds([verbInt])).toEqual([VERB_UUID_3])
|
||||
})
|
||||
|
||||
it('fails loudly when the entityIdMapper is not wired', async () => {
|
||||
const bare = new GraphAdjacencyIndex(storage)
|
||||
await expect(bare.getNeighbors(1n)).rejects.toThrow(/entityIdMapper not wired/)
|
||||
await bare.close()
|
||||
})
|
||||
})
|
||||
|
||||
describe('Coordinator — relate/related end-to-end over the BigInt boundary', () => {
|
||||
let brain: Brainy
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||
await brain.init()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
it('relate → getRelations → neighbors round-trips (public API unchanged)', async () => {
|
||||
const personId = await brain.add({
|
||||
data: { name: 'Ada' },
|
||||
type: NounType.Person
|
||||
})
|
||||
const projectId = await brain.add({
|
||||
data: { name: 'Analytical Engine' },
|
||||
type: NounType.Thing
|
||||
})
|
||||
|
||||
const relId = await brain.relate({
|
||||
from: personId,
|
||||
to: projectId,
|
||||
type: VerbType.WorksWith
|
||||
})
|
||||
|
||||
// getRelations resolves verb ints back to verb-id strings internally.
|
||||
const relations = await brain.getRelations({ from: personId })
|
||||
expect(relations).toHaveLength(1)
|
||||
expect(relations[0].id).toBe(relId)
|
||||
expect(relations[0].to).toBe(projectId)
|
||||
|
||||
// neighbors() routes through getNeighbors(bigint) + int → UUID mapping.
|
||||
const out = await brain.neighbors(personId, { direction: 'outgoing' })
|
||||
expect(out).toContain(projectId)
|
||||
|
||||
const incoming = await brain.neighbors(projectId, { direction: 'incoming' })
|
||||
expect(incoming).toContain(personId)
|
||||
})
|
||||
|
||||
it('relate feeds the verb-int warm cache; duplicate detection still works', async () => {
|
||||
const a = await brain.add({ data: { name: 'A' }, type: NounType.Thing })
|
||||
const b = await brain.add({ data: { name: 'B' }, type: NounType.Thing })
|
||||
|
||||
const first = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
||||
|
||||
// The AddToGraphIndexOperation callback recorded the verb int → id pair.
|
||||
const warmCache: Map<bigint, string> = (brain as any).verbIntWarmCache
|
||||
expect([...warmCache.values()]).toContain(first)
|
||||
|
||||
// Duplicate check runs through getVerbIdsBySource(bigint) + resolution.
|
||||
const duplicate = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
||||
expect(duplicate).toBe(first)
|
||||
})
|
||||
|
||||
it('unrelate removes the edge from BigInt-boundary reads', async () => {
|
||||
const a = await brain.add({ data: { name: 'A' }, type: NounType.Thing })
|
||||
const b = await brain.add({ data: { name: 'B' }, type: NounType.Thing })
|
||||
|
||||
const relId = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
|
||||
await brain.unrelate(relId)
|
||||
|
||||
const relations = await brain.getRelations({ from: a })
|
||||
expect(relations).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('neighbors() of an unknown UUID returns empty without error', async () => {
|
||||
const result = await brain.neighbors('00000000-0000-4000-8000-000000000000')
|
||||
expect(result).toEqual([])
|
||||
})
|
||||
|
||||
it('relate() rejects a caller-supplied verb id with a teaching error (8.0 contract)', async () => {
|
||||
const a = await brain.add({ data: { name: 'A' }, type: NounType.Thing })
|
||||
const b = await brain.add({ data: { name: 'B' }, type: NounType.Thing })
|
||||
|
||||
// RelateParams has no `id` field — untyped callers passing one used to be
|
||||
// silently ignored. 8.0 teaches: verb ids are brainy-generated UUIDs.
|
||||
await expect(
|
||||
brain.relate({ from: a, to: b, type: VerbType.RelatedTo, id: 'my-custom-id' } as any)
|
||||
).rejects.toThrow(/Verb ids are UUIDs by contract in 8\.0/)
|
||||
})
|
||||
})
|
||||
|
|
@ -1,12 +1,14 @@
|
|||
/**
|
||||
* GraphAdjacencyIndex Pagination Tests (v5.8.0)
|
||||
* GraphAdjacencyIndex Pagination Tests (v5.8.0, updated for the 8.0 u64 contract)
|
||||
*
|
||||
* Tests for pagination support in GraphIndex methods:
|
||||
* - getNeighbors() with limit/offset
|
||||
* - getVerbIdsBySource() with limit/offset
|
||||
* - getVerbIdsByTarget() with limit/offset
|
||||
*
|
||||
* Verifies backward compatibility and new pagination features
|
||||
* 8.0 BigInt boundary: entity ints in (resolved via the metadata index's
|
||||
* idMapper), entity/verb ints out (`bigint[]`). Entity ints map back to UUIDs
|
||||
* via `idMapper.getUuid(Number(int))`; verb ints via `verbIntsToIds()`.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
|
|
@ -18,6 +20,25 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
let centralId: string
|
||||
let neighborIds: string[]
|
||||
|
||||
/** The brain's graph index (BigInt provider contract). */
|
||||
const graphIndex = () => (brain as any).graphIndex
|
||||
|
||||
/** The shared UUID ↔ int resolver. */
|
||||
const idMapper = () => (brain as any).metadataIndex.getIdMapper()
|
||||
|
||||
/** Resolve a UUID to its entity int (must already be mapped). */
|
||||
const entityInt = (uuid: string): bigint => {
|
||||
const intId = idMapper().getInt(uuid)
|
||||
expect(intId).toBeDefined()
|
||||
return BigInt(intId!)
|
||||
}
|
||||
|
||||
/** Map returned entity ints back to UUIDs. */
|
||||
const intsToUuids = (ints: bigint[]): string[] =>
|
||||
ints
|
||||
.map((i) => idMapper().getUuid(Number(i)))
|
||||
.filter((u: string | undefined): u is string => u !== undefined)
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({ requireSubtype: false })
|
||||
await brain.init()
|
||||
|
|
@ -47,50 +68,42 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
})
|
||||
|
||||
describe('getNeighbors() Pagination', () => {
|
||||
it('should return all neighbors without pagination (backward compatible)', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const neighbors = await graphIndex.getNeighbors(centralId)
|
||||
it('should return all neighbors without pagination', async () => {
|
||||
const neighborInts = await graphIndex().getNeighbors(entityInt(centralId))
|
||||
const neighbors = intsToUuids(neighborInts)
|
||||
|
||||
// Should return all 50 neighbors
|
||||
expect(neighbors).toHaveLength(50)
|
||||
expect(neighbors.every((id: string) => neighborIds.includes(id))).toBe(true)
|
||||
})
|
||||
|
||||
it('should return all outgoing neighbors with direction parameter (backward compatible)', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
// Old API: direction as string
|
||||
const neighbors = await graphIndex.getNeighbors(centralId, 'out')
|
||||
it('should return all outgoing neighbors with direction option', async () => {
|
||||
const neighbors = await graphIndex().getNeighbors(entityInt(centralId), { direction: 'out' })
|
||||
|
||||
expect(neighbors).toHaveLength(50)
|
||||
})
|
||||
|
||||
it('should paginate with limit only', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const page1 = await graphIndex.getNeighbors(centralId, { limit: 10 })
|
||||
const page1Ints = await graphIndex().getNeighbors(entityInt(centralId), { limit: 10 })
|
||||
const page1 = intsToUuids(page1Ints)
|
||||
|
||||
expect(page1).toHaveLength(10)
|
||||
expect(page1.every((id: string) => neighborIds.includes(id))).toBe(true)
|
||||
})
|
||||
|
||||
it('should paginate with offset only', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const all = await graphIndex.getNeighbors(centralId)
|
||||
const offsetResults = await graphIndex.getNeighbors(centralId, { offset: 10 })
|
||||
const all = await graphIndex().getNeighbors(entityInt(centralId))
|
||||
const offsetResults = await graphIndex().getNeighbors(entityInt(centralId), { offset: 10 })
|
||||
|
||||
// Offset 10 should return all after first 10
|
||||
expect(offsetResults).toHaveLength(all.length - 10)
|
||||
})
|
||||
|
||||
it('should paginate with both limit and offset', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const page1 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 0 })
|
||||
const page2 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 10 })
|
||||
const page3 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 20 })
|
||||
const central = entityInt(centralId)
|
||||
const page1: bigint[] = await graphIndex().getNeighbors(central, { limit: 10, offset: 0 })
|
||||
const page2: bigint[] = await graphIndex().getNeighbors(central, { limit: 10, offset: 10 })
|
||||
const page3: bigint[] = await graphIndex().getNeighbors(central, { limit: 10, offset: 20 })
|
||||
|
||||
// Each page should have 10 results
|
||||
expect(page1).toHaveLength(10)
|
||||
|
|
@ -98,39 +111,32 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
expect(page3).toHaveLength(10)
|
||||
|
||||
// Pages should not overlap
|
||||
const page1Set = new Set(page1)
|
||||
const page2Set = new Set(page2)
|
||||
const page3Set = new Set(page3)
|
||||
|
||||
const overlap12 = page1.filter(id => page2Set.has(id))
|
||||
const overlap23 = page2.filter(id => page3Set.has(id))
|
||||
const overlap12 = page1.filter((id) => page2Set.has(id))
|
||||
const overlap23 = page2.filter((id) => page3Set.has(id))
|
||||
|
||||
expect(overlap12).toHaveLength(0)
|
||||
expect(overlap23).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('should handle offset beyond total count', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const results = await graphIndex.getNeighbors(centralId, { offset: 100 })
|
||||
const results = await graphIndex().getNeighbors(entityInt(centralId), { offset: 100 })
|
||||
|
||||
// Offset 100 > 50 total → empty array
|
||||
expect(results).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('should handle limit = 0', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const results = await graphIndex.getNeighbors(centralId, { limit: 0 })
|
||||
const results = await graphIndex().getNeighbors(entityInt(centralId), { limit: 0 })
|
||||
|
||||
expect(results).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('should paginate with direction filter', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
// Get first 10 outgoing neighbors
|
||||
const page1 = await graphIndex.getNeighbors(centralId, {
|
||||
const page1 = await graphIndex().getNeighbors(entityInt(centralId), {
|
||||
direction: 'out',
|
||||
limit: 10,
|
||||
offset: 0
|
||||
|
|
@ -140,8 +146,6 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
})
|
||||
|
||||
it('should handle pagination with incoming direction', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
// Create some incoming relationships
|
||||
const sourceId = await brain.add({
|
||||
data: { name: 'Source' },
|
||||
|
|
@ -155,7 +159,7 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
})
|
||||
|
||||
// Get incoming neighbors with pagination
|
||||
const incoming = await graphIndex.getNeighbors(centralId, {
|
||||
const incoming = await graphIndex().getNeighbors(entityInt(centralId), {
|
||||
direction: 'in',
|
||||
limit: 10
|
||||
})
|
||||
|
|
@ -165,38 +169,36 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
})
|
||||
|
||||
describe('getVerbIdsBySource() Pagination', () => {
|
||||
it('should return all verb IDs without pagination (backward compatible)', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
it('should return all verb ints without pagination and resolve them back to ids', async () => {
|
||||
const verbInts: bigint[] = await graphIndex().getVerbIdsBySource(entityInt(centralId))
|
||||
|
||||
const verbIds = await graphIndex.getVerbIdsBySource(centralId)
|
||||
// Should return all 50 verb ints
|
||||
expect(verbInts).toHaveLength(50)
|
||||
|
||||
// Should return all 50 verb IDs
|
||||
// Every int must resolve back to a verb-id string
|
||||
const verbIds: (string | null)[] = await graphIndex().verbIntsToIds(verbInts)
|
||||
expect(verbIds).toHaveLength(50)
|
||||
expect(verbIds.every((id) => typeof id === 'string')).toBe(true)
|
||||
})
|
||||
|
||||
it('should paginate verb IDs with limit', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const page1 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10 })
|
||||
it('should paginate verb ints with limit', async () => {
|
||||
const page1 = await graphIndex().getVerbIdsBySource(entityInt(centralId), { limit: 10 })
|
||||
|
||||
expect(page1).toHaveLength(10)
|
||||
})
|
||||
|
||||
it('should paginate verb IDs with offset', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const all = await graphIndex.getVerbIdsBySource(centralId)
|
||||
const offsetResults = await graphIndex.getVerbIdsBySource(centralId, { offset: 10 })
|
||||
it('should paginate verb ints with offset', async () => {
|
||||
const all = await graphIndex().getVerbIdsBySource(entityInt(centralId))
|
||||
const offsetResults = await graphIndex().getVerbIdsBySource(entityInt(centralId), { offset: 10 })
|
||||
|
||||
expect(offsetResults).toHaveLength(all.length - 10)
|
||||
})
|
||||
|
||||
it('should paginate verb IDs with limit and offset', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
const page1 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 0 })
|
||||
const page2 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 10 })
|
||||
const page3 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 20 })
|
||||
it('should paginate verb ints with limit and offset', async () => {
|
||||
const central = entityInt(centralId)
|
||||
const page1: bigint[] = await graphIndex().getVerbIdsBySource(central, { limit: 10, offset: 0 })
|
||||
const page2: bigint[] = await graphIndex().getVerbIdsBySource(central, { limit: 10, offset: 10 })
|
||||
const page3: bigint[] = await graphIndex().getVerbIdsBySource(central, { limit: 10, offset: 20 })
|
||||
|
||||
// Each page should have 10 results
|
||||
expect(page1).toHaveLength(10)
|
||||
|
|
@ -209,34 +211,28 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
expect(unique.size).toBe(30) // All unique
|
||||
})
|
||||
|
||||
it('should handle pagination edge cases for verb IDs', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
it('should handle pagination edge cases for verb ints', async () => {
|
||||
// Offset beyond total
|
||||
const beyondTotal = await graphIndex.getVerbIdsBySource(centralId, { offset: 100 })
|
||||
const beyondTotal = await graphIndex().getVerbIdsBySource(entityInt(centralId), { offset: 100 })
|
||||
expect(beyondTotal).toHaveLength(0)
|
||||
|
||||
// Limit = 0
|
||||
const zeroLimit = await graphIndex.getVerbIdsBySource(centralId, { limit: 0 })
|
||||
const zeroLimit = await graphIndex().getVerbIdsBySource(entityInt(centralId), { limit: 0 })
|
||||
expect(zeroLimit).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('getVerbIdsByTarget() Pagination', () => {
|
||||
it('should return all verb IDs targeting an entity (backward compatible)', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
it('should return all verb ints targeting an entity', async () => {
|
||||
// Pick a neighbor that's a target of relationships
|
||||
const targetId = neighborIds[0]
|
||||
const verbIds = await graphIndex.getVerbIdsByTarget(targetId)
|
||||
const verbInts = await graphIndex().getVerbIdsByTarget(entityInt(targetId))
|
||||
|
||||
// Should have at least 1 (the relationship from central)
|
||||
expect(verbIds.length).toBeGreaterThanOrEqual(1)
|
||||
expect(verbInts.length).toBeGreaterThanOrEqual(1)
|
||||
})
|
||||
|
||||
it('should paginate verb IDs by target with limit', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
it('should paginate verb ints by target with limit', async () => {
|
||||
// Create entity with many incoming relationships
|
||||
const popularTarget = await brain.add({
|
||||
data: { name: 'Popular Target' },
|
||||
|
|
@ -257,28 +253,29 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
}
|
||||
|
||||
// Paginate
|
||||
const page1 = await graphIndex.getVerbIdsByTarget(popularTarget, { limit: 10 })
|
||||
const page2 = await graphIndex.getVerbIdsByTarget(popularTarget, { limit: 10, offset: 10 })
|
||||
const target = entityInt(popularTarget)
|
||||
const page1: bigint[] = await graphIndex().getVerbIdsByTarget(target, { limit: 10 })
|
||||
const page2: bigint[] = await graphIndex().getVerbIdsByTarget(target, { limit: 10, offset: 10 })
|
||||
|
||||
expect(page1).toHaveLength(10)
|
||||
expect(page2).toHaveLength(10)
|
||||
|
||||
// No overlap
|
||||
const overlap = page1.filter((id: string) => page2.includes(id))
|
||||
const overlap = page1.filter((id) => page2.includes(id))
|
||||
expect(overlap).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Performance with Pagination', () => {
|
||||
it('should maintain sub-5ms performance with pagination', async () => {
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
const central = entityInt(centralId)
|
||||
|
||||
// Multiple paginated queries should be fast
|
||||
const startTime = performance.now()
|
||||
|
||||
await graphIndex.getNeighbors(centralId, { limit: 10, offset: 0 })
|
||||
await graphIndex.getNeighbors(centralId, { limit: 10, offset: 10 })
|
||||
await graphIndex.getNeighbors(centralId, { limit: 10, offset: 20 })
|
||||
await graphIndex().getNeighbors(central, { limit: 10, offset: 0 })
|
||||
await graphIndex().getNeighbors(central, { limit: 10, offset: 10 })
|
||||
await graphIndex().getNeighbors(central, { limit: 10, offset: 20 })
|
||||
|
||||
const elapsed = performance.now() - startTime
|
||||
|
||||
|
|
@ -310,12 +307,11 @@ describe('GraphAdjacencyIndex Pagination', () => {
|
|||
})
|
||||
}
|
||||
|
||||
const graphIndex = (brain as any).graphIndex
|
||||
|
||||
// Paginate in chunks of 25
|
||||
const chunks: string[][] = []
|
||||
const hubInt = entityInt(hub)
|
||||
const chunks: bigint[][] = []
|
||||
for (let offset = 0; offset < 100; offset += 25) {
|
||||
const chunk = await graphIndex.getNeighbors(hub, {
|
||||
const chunk = await graphIndex().getNeighbors(hubInt, {
|
||||
direction: 'out',
|
||||
limit: 25,
|
||||
offset
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
|
||||
it('flushBuffer writes segment bytes as a raw blob at the cortex-shared key', async () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i * 10 })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
|
|
@ -63,9 +63,9 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
|
||||
it('flushBuffer writes the DELETED bitmap as a raw blob too (not an envelope)', async () => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i })
|
||||
}
|
||||
store.removeEntity(idMapper.getInt('u-1')!)
|
||||
store.removeEntity(BigInt(idMapper.getInt('u-1')!))
|
||||
await store.flush()
|
||||
|
||||
const rawDel = await (storage as any).loadBinaryBlob('_column_index/score/DELETED')
|
||||
|
|
@ -135,7 +135,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
const readStore = new ColumnStore()
|
||||
await readStore.init(storage, idMapper)
|
||||
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i)))
|
||||
expect(sortedUuids).toEqual(['legacy-a', 'legacy-b', 'legacy-c'])
|
||||
await readStore.close()
|
||||
})
|
||||
|
|
@ -153,15 +153,15 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
|
||||
// Add a couple of live entities so a manifest exists (init only loads
|
||||
// DELETED for fields with a manifest on disk).
|
||||
store.addEntity(idMapper.getOrAssign('alive-1'), { score: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('alive-2'), { score: 200 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('alive-1')), { score: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('alive-2')), { score: 200 })
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
||||
const readStore = new ColumnStore()
|
||||
await readStore.init(storage, idMapper)
|
||||
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i)))
|
||||
|
||||
// The dead entity must NOT appear; the alive ones must.
|
||||
expect(sortedUuids).not.toContain('dead')
|
||||
|
|
@ -176,7 +176,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
|
||||
it('write (new format) → re-init → read returns the same data', async () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i * 10 })
|
||||
}
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
|
@ -184,7 +184,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
|
|||
const readStore = new ColumnStore()
|
||||
await readStore.init(storage, idMapper)
|
||||
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
|
||||
const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i)))
|
||||
expect(sortedUuids).toEqual(['u-0', 'u-1', 'u-2', 'u-3', 'u-4'])
|
||||
await readStore.close()
|
||||
})
|
||||
|
|
|
|||
|
|
@ -35,48 +35,48 @@ describe('ColumnStore', () => {
|
|||
|
||||
describe('sortTopK', () => {
|
||||
it('returns entities sorted by numeric field ascending', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['b', 'c', 'a'])
|
||||
})
|
||||
|
||||
it('returns entities sorted by numeric field descending', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['a', 'c', 'b'])
|
||||
})
|
||||
|
||||
it('respects limit K', async () => {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`e${i}`), { createdAt: i * 1000 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { createdAt: i * 1000 })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'desc', 5)
|
||||
expect(result).toHaveLength(5)
|
||||
// Should be the 5 highest timestamps
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['e49', 'e48', 'e47', 'e46', 'e45'])
|
||||
})
|
||||
|
||||
it('sorts string fields lexicographically', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'pending' })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { status: 'inactive' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'pending' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { status: 'inactive' })
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('status', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['b', 'c', 'a']) // active, inactive, pending
|
||||
})
|
||||
|
||||
|
|
@ -97,16 +97,16 @@ describe('ColumnStore', () => {
|
|||
const idC = idMapper.getOrAssign('c')
|
||||
const idD = idMapper.getOrAssign('d')
|
||||
|
||||
store.addEntity(idA, { createdAt: 400 })
|
||||
store.addEntity(idB, { createdAt: 100 })
|
||||
store.addEntity(idC, { createdAt: 300 })
|
||||
store.addEntity(idD, { createdAt: 200 })
|
||||
store.addEntity(BigInt(idA), { createdAt: 400 })
|
||||
store.addEntity(BigInt(idB), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idC), { createdAt: 300 })
|
||||
store.addEntity(BigInt(idD), { createdAt: 200 })
|
||||
await store.flush()
|
||||
|
||||
// Filter: only B and C
|
||||
const bitmap = new RoaringBitmap32([idB, idC])
|
||||
const result = await store.filteredSortTopK(bitmap, 'createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['c', 'b']) // 300, 100
|
||||
})
|
||||
})
|
||||
|
|
@ -121,9 +121,9 @@ describe('ColumnStore', () => {
|
|||
const idB = idMapper.getOrAssign('b')
|
||||
const idC = idMapper.getOrAssign('c')
|
||||
|
||||
store.addEntity(idA, { status: 'active' })
|
||||
store.addEntity(idB, { status: 'inactive' })
|
||||
store.addEntity(idC, { status: 'active' })
|
||||
store.addEntity(BigInt(idA), { status: 'active' })
|
||||
store.addEntity(BigInt(idB), { status: 'inactive' })
|
||||
store.addEntity(BigInt(idC), { status: 'active' })
|
||||
await store.flush()
|
||||
|
||||
const bitmap = await store.filter('status', 'active')
|
||||
|
|
@ -133,7 +133,7 @@ describe('ColumnStore', () => {
|
|||
})
|
||||
|
||||
it('returns empty bitmap for no match', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' })
|
||||
await store.flush()
|
||||
|
||||
const bitmap = await store.filter('status', 'deleted')
|
||||
|
|
@ -150,7 +150,7 @@ describe('ColumnStore', () => {
|
|||
const ids: number[] = []
|
||||
for (let i = 0; i < 10; i++) {
|
||||
ids.push(idMapper.getOrAssign(`e${i}`))
|
||||
store.addEntity(ids[i], { price: i * 10 })
|
||||
store.addEntity(BigInt(ids[i]), { price: i * 10 })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
|
|
@ -174,16 +174,16 @@ describe('ColumnStore', () => {
|
|||
const idB = idMapper.getOrAssign('b')
|
||||
const idC = idMapper.getOrAssign('c')
|
||||
|
||||
store.addEntity(idA, { createdAt: 100 })
|
||||
store.addEntity(idB, { createdAt: 200 })
|
||||
store.addEntity(idC, { createdAt: 300 })
|
||||
store.addEntity(BigInt(idA), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idB), { createdAt: 200 })
|
||||
store.addEntity(BigInt(idC), { createdAt: 300 })
|
||||
await store.flush()
|
||||
|
||||
store.removeEntity(idB)
|
||||
store.removeEntity(BigInt(idB))
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('createdAt', 'asc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['a', 'c'])
|
||||
})
|
||||
})
|
||||
|
|
@ -196,12 +196,12 @@ describe('ColumnStore', () => {
|
|||
it('sorts correctly across multiple L0 segments', async () => {
|
||||
// Flush threshold is 10, so 25 entities creates 3 segments
|
||||
for (let i = 0; i < 25; i++) {
|
||||
store.addEntity(idMapper.getOrAssign(`e${i}`), { ts: i })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { ts: i })
|
||||
}
|
||||
await store.flush()
|
||||
|
||||
const result = await store.sortTopK('ts', 'desc', 5)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['e24', 'e23', 'e22', 'e21', 'e20'])
|
||||
})
|
||||
})
|
||||
|
|
@ -212,9 +212,9 @@ describe('ColumnStore', () => {
|
|||
|
||||
describe('persistence', () => {
|
||||
it('survives close + reload', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 })
|
||||
await store.flush()
|
||||
await store.close()
|
||||
|
||||
|
|
@ -223,7 +223,7 @@ describe('ColumnStore', () => {
|
|||
await store2.init(storage, idMapper)
|
||||
|
||||
const result = await store2.sortTopK('createdAt', 'desc', 10)
|
||||
const uuids = result.map(id => idMapper.getUuid(id))
|
||||
const uuids = result.map(id => idMapper.getUuid(Number(id)))
|
||||
expect(uuids).toEqual(['a', 'c', 'b'])
|
||||
|
||||
await store2.close()
|
||||
|
|
@ -239,8 +239,8 @@ describe('ColumnStore', () => {
|
|||
const idA = idMapper.getOrAssign('docA')
|
||||
const idB = idMapper.getOrAssign('docB')
|
||||
|
||||
store.addEntity(idA, { __words__: ['machine', 'learning', 'algorithm'] })
|
||||
store.addEntity(idB, { __words__: ['neural', 'network', 'machine'] })
|
||||
store.addEntity(BigInt(idA), { __words__: ['machine', 'learning', 'algorithm'] })
|
||||
store.addEntity(BigInt(idB), { __words__: ['neural', 'network', 'machine'] })
|
||||
await store.flush()
|
||||
|
||||
// Point filter: "machine" should match both
|
||||
|
|
@ -267,10 +267,10 @@ describe('ColumnStore', () => {
|
|||
|
||||
describe('getFilterValues', () => {
|
||||
it('returns distinct values sorted', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { status: 'pending' })
|
||||
store.addEntity(idMapper.getOrAssign('b'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('c'), { status: 'active' })
|
||||
store.addEntity(idMapper.getOrAssign('d'), { status: 'inactive' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'pending' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('c')), { status: 'active' })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('d')), { status: 'inactive' })
|
||||
await store.flush()
|
||||
|
||||
const values = await store.getFilterValues('status')
|
||||
|
|
@ -288,7 +288,7 @@ describe('ColumnStore', () => {
|
|||
})
|
||||
|
||||
it('returns true after adding data', async () => {
|
||||
store.addEntity(idMapper.getOrAssign('a'), { createdAt: 100 })
|
||||
store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 100 })
|
||||
expect(store.hasField('createdAt')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue