From 2427bb79601a4424d7ef327397e9db9249b0f171 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 10 Jun 2026 10:45:45 -0700 Subject: [PATCH] =?UTF-8?q?feat(8.0):=20u64=20BigInt=20graph=20provider=20?= =?UTF-8?q?contract=20=E2=80=94=20punch=20list=20a-d,g,h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GraphIndexProvider now speaks BigInt at the boundary (D.2 mirror): - getNeighbors/getVerbIdsBySource/getVerbIdsByTarget take entity ints and return entity/verb ints as bigint[] - new REQUIRED verbIntsToIds(bigint[]) batch reverse resolver (L.7 identity-fingerprint design — verb ids are UUIDs by contract, so the provider-side interning is losslessly reversible) - addVerb(verb, sourceInt, targetInt) returns the interned verb int; removeVerb(verbId) joins the contract Coordinator (brainy.ts) owns ALL UUID <-> int conversion: getOrAssign on writes, getInt on reads (unmapped UUID -> empty result without calling the provider), getUuid / verbIntsToIds on returns, plus a bounded ~100k-entry insertion-order warm cache for verb-int -> verb-id pairs fed by addVerb returns and resolver results. GraphVerb gains derived sourceInt/targetInt (populated at add time, never persisted). findConnectedSubtype gains a native fast path that routes single-type single-subtype outgoing BFS through the provider when available. JS GraphAdjacencyIndex satisfies the contract while staying string/u32-keyed internally: entity ints resolve through the shared entity-id mapper (threaded in by the coordinator on init/fork/checkout), verb ints come from an in-process append-only interning map re-derived from storage on rebuild/cold-start. ColumnStoreProvider widens the same way: addEntity/ removeEntity take bigint, sortTopK/filteredSortTopK return bigint[]. relate() now rejects a caller-supplied id with a teaching error — verb ids are brainy-generated UUIDs by contract in 8.0 (previously a passed id was silently ignored). No Roaring64 provider-boundary decode site exists yet; the JS-internal column store stays Roaring32 and the Treemap decoder lands with the first consumer of provider-returned filter buffers. Public brain API unchanged. 1413 tests green (+10 new BigInt contract tests). --- src/brainy.ts | 308 ++++++++++++++-- src/coreTypes.ts | 18 +- src/graph/graphAdjacencyIndex.ts | 341 ++++++++++++------ src/indexes/columnStore/ColumnStore.ts | 46 ++- src/indexes/columnStore/types.ts | 24 +- src/plugin.ts | 82 ++++- src/storage/baseStorage.ts | 61 +++- src/transaction/operations/IndexOperations.ts | 43 ++- src/types/graphTypes.ts | 17 +- src/utils/metadataIndex.ts | 11 +- src/utils/paramValidation.ts | 15 + .../graph-scale-performance.test.ts | 28 +- .../relate-duplicate-optimization.test.ts | 8 +- tests/unit/graph/bigint-contract.test.ts | 230 ++++++++++++ .../unit/graph/graphIndex-pagination.test.ts | 158 ++++---- .../column-store-interchange.test.ts | 18 +- .../indexes/columnStore/column-store.test.ts | 84 ++--- 17 files changed, 1164 insertions(+), 328 deletions(-) create mode 100644 tests/unit/graph/bigint-contract.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 5248e217..6afc4ec4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -164,6 +164,17 @@ export class Brainy implements BrainyInterface { private distance: DistanceFunction private config: Required + /** + * Bounded warm cache for verb-int → verb-id resolution (8.0 u64 contract). + * Fed by `addVerb` returns and `verbIntsToIds` results; evicted in insertion + * order once {@link Brainy.VERB_INT_WARM_CACHE_MAX} is exceeded. Pure + * optimization — the graph-index provider owns the durable interning, so a + * miss just means one extra `verbIntsToIds` round trip. + */ + private readonly verbIntWarmCache = new Map() + /** Cap for {@link Brainy.verbIntWarmCache} (~100k entries ≈ a few MB). */ + private static readonly VERB_INT_WARM_CACHE_MAX = 100_000 + // Distributed components (optional) private coordinator?: DistributedCoordinator private shardManager?: ShardManager @@ -602,6 +613,10 @@ export class Brainy implements BrainyInterface { this.graphIndex = graphIndex } + // 8.0 u64 contract: thread the shared UUID ↔ int resolver into the JS + // graph index and the storage layer's verb read paths. + this.wireGraphIdResolver() + // Wire the connections codec (2.4.0 #3). When the graph:compression // provider is registered AND the metadata index exposes a stable // idMapper, inject a codec that encodes HNSW connections as @@ -1741,9 +1756,11 @@ export class Brainy implements BrainyInterface { // Operations 4+: Delete all related verbs atomically for (const verb of allVerbs) { - // Remove from graph index + // Remove from graph index (endpoint ints resolved up front so a + // rollback can re-add through the BigInt addVerb contract) + const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) tx.addOperation( - new RemoveFromGraphIndexOperation(this.graphIndex, verb as any) + new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt) ) // Delete verb metadata tx.addOperation( @@ -1768,6 +1785,126 @@ export class Brainy implements BrainyInterface { // ============= RELATIONSHIP OPERATIONS ============= + // --- 8.0 u64 boundary helpers ------------------------------------------- + // UUID ↔ int conversion happens ONCE here at the coordinator boundary: + // `getOrAssign` on writes, `getInt` on reads (undefined → the entity was + // never mapped, i.e. it has no relations — return empty without calling the + // provider). Provider returns convert back via `getUuid` (entities) and + // `verbIntsToIds` + the warm cache (verbs). `Number(bigint)` narrowing is + // lossless under the shipped EntityIdSpaceExceeded u32 guard. + + /** + * Resolve a UUID to its entity int for a READ — `undefined` means the + * entity was never mapped and therefore has no relations. + */ + private graphEntityInt(uuid: string): bigint | undefined { + const intId = this.metadataIndex.getIdMapper().getInt(uuid) + return intId === undefined ? undefined : BigInt(intId) + } + + /** + * Resolve a verb's endpoint UUIDs to entity ints for a WRITE + * (`getOrAssign`), mirroring them onto `verb.sourceInt`/`verb.targetInt` + * (derived state — never persisted to storage JSON) before the verb is + * handed to the graph-index provider. Accepts any verb shape carrying + * endpoint UUIDs (`GraphVerb`, `HNSWVerbWithMetadata`, …). + */ + private resolveVerbEndpointInts( + verb: Pick & { sourceInt?: bigint; targetInt?: bigint } + ): { sourceInt: bigint; targetInt: bigint } { + const idMapper = this.metadataIndex.getIdMapper() + const sourceInt = BigInt(idMapper.getOrAssign(verb.sourceId)) + const targetInt = BigInt(idMapper.getOrAssign(verb.targetId)) + verb.sourceInt = sourceInt + verb.targetInt = targetInt + return { sourceInt, targetInt } + } + + /** + * Convert provider-returned entity ints back to UUIDs, dropping ints the + * mapper no longer knows (deleted entities). + */ + private entityIntsToUuids(entityInts: bigint[]): string[] { + const idMapper = this.metadataIndex.getIdMapper() + const uuids: string[] = [] + for (const entityInt of entityInts) { + const uuid = idMapper.getUuid(Number(entityInt)) + if (uuid !== undefined) uuids.push(uuid) + } + return uuids + } + + /** + * Record one verb-int → verb-id pair in the bounded warm cache, evicting + * the oldest entries (insertion order) past the cap. + */ + private cacheVerbInt(verbInt: bigint, verbId: string): void { + if (!this.verbIntWarmCache.has(verbInt) && + this.verbIntWarmCache.size >= Brainy.VERB_INT_WARM_CACHE_MAX) { + // Evict oldest insertions until under cap (single eviction in the + // common case; loop guards against future cap reductions). + for (const oldest of this.verbIntWarmCache.keys()) { + this.verbIntWarmCache.delete(oldest) + if (this.verbIntWarmCache.size < Brainy.VERB_INT_WARM_CACHE_MAX) break + } + } + this.verbIntWarmCache.set(verbInt, verbId) + } + + /** + * Resolve provider-returned verb ints to verb-id strings: warm cache first, + * then one batched `verbIntsToIds` call for the misses (which also refills + * the cache). Unknown ints are dropped. + */ + private async resolveVerbIntsToIds(verbInts: bigint[]): Promise { + if (verbInts.length === 0) return [] + + const resolved = new Array(verbInts.length) + const missIndices: number[] = [] + const missInts: bigint[] = [] + + for (let i = 0; i < verbInts.length; i++) { + const cached = this.verbIntWarmCache.get(verbInts[i]) + if (cached !== undefined) { + resolved[i] = cached + } else { + missIndices.push(i) + missInts.push(verbInts[i]) + } + } + + if (missInts.length > 0) { + const ids = await this.graphIndex.verbIntsToIds(missInts) + for (let j = 0; j < missInts.length; j++) { + const id = ids[j] + if (id !== null) { + resolved[missIndices[j]] = id + this.cacheVerbInt(missInts[j], id) + } + } + } + + return resolved.filter((id): id is string => id !== undefined) + } + + /** + * UUID-level neighbor lookup over the BigInt provider contract: resolves + * the anchor via `getInt` (unmapped → empty), then maps returned entity + * ints back to UUIDs. Shared by the traversal paths and the + * TripleIntelligenceSystem adapter. + */ + private async getNeighborUuids( + uuid: string, + options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number } + ): Promise { + const entityInt = this.graphEntityInt(uuid) + if (entityInt === undefined) return [] + const neighborInts = await this.graphIndex.getNeighbors(entityInt, options) + return this.entityIntsToUuids(neighborInts) + } + + // ------------------------------------------------------------------------- + /** * Create a relationship (verb) between two entities * @@ -1790,7 +1927,12 @@ export class Brainy implements BrainyInterface { * @param params.service - Multi-tenancy service name * @param params.confidence - Relationship certainty 0-1 * @param params.evidence - Why this relationship exists - * @returns Promise that resolves to the relationship ID + * @returns Promise that resolves to the relationship ID. **Contract (8.0):** + * relationship ids are always UUIDs (36-char 8-4-4-4-12 hex). Brainy + * generates every verb id itself (`relate()` takes no custom id), so the + * UUID shape is a guarantee, not a coincidence — graph-index providers may + * key their verb-int interning on the raw UUID bytes for lossless + * reverse lookup. * * @example * // Basic relationship creation @@ -1918,7 +2060,11 @@ export class Brainy implements BrainyInterface { // This prevents infinite loops where same relationship is created repeatedly // Bug #1 showed incrementing verb counts (7→8→9...) indicating duplicates // OPTIMIZATION: Use GraphAdjacencyIndex for O(log n) lookup instead of O(n) storage scan - const verbIds = await this.graphIndex.getVerbIdsBySource(params.from) + // 8.0 BigInt boundary: unmapped source → no existing relations to check. + const dupSourceInt = this.graphEntityInt(params.from) + const verbIds = dupSourceInt === undefined + ? [] + : await this.resolveVerbIntsToIds(await this.graphIndex.getVerbIdsBySource(dupSourceInt)) // Batch-load verbs for 5x faster duplicate checking on GCS // GCS: 5 verbs = 1×50ms vs 5×50ms = 250ms (5x faster) @@ -1969,6 +2115,10 @@ export class Brainy implements BrainyInterface { createdAt: Date.now() } + // 8.0 BigInt boundary: resolve endpoint ints ONCE (getOrAssign — both + // entities were existence-checked above) and mirror them onto the verb. + const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) + // Execute atomically with transaction system await this.transactionManager.executeTransaction(async (tx) => { // Operation 1: Save verb vector data @@ -1990,7 +2140,10 @@ export class Brainy implements BrainyInterface { // Operation 3: Add to graph index for O(1) lookups tx.addOperation( - new AddToGraphIndexOperation(this.graphIndex, verb) + new AddToGraphIndexOperation( + this.graphIndex, verb, sourceInt, targetInt, + (verbInt) => this.cacheVerbInt(verbInt, id) + ) ) // Create bidirectional if requested @@ -2000,7 +2153,10 @@ export class Brainy implements BrainyInterface { ...verb, id: reverseId, sourceId: params.to, - targetId: params.from + targetId: params.from, + // Endpoints swap, so the derived ints swap with them. + sourceInt: targetInt, + targetInt: sourceInt } // Operation 4: Save reverse verb vector data @@ -2022,7 +2178,10 @@ export class Brainy implements BrainyInterface { // Operation 6: Add reverse relationship to graph index tx.addOperation( - new AddToGraphIndexOperation(this.graphIndex, reverseVerb) + new AddToGraphIndexOperation( + this.graphIndex, reverseVerb, targetInt, sourceInt, + (verbInt) => this.cacheVerbInt(verbInt, reverseId) + ) ) } }) @@ -2053,12 +2212,18 @@ export class Brainy implements BrainyInterface { // Get verb data before deletion for rollback const verb = await this.storage.getVerb(id) + // 8.0 BigInt boundary: resolve endpoint ints before the transaction so + // a rollback can re-add through the BigInt addVerb contract. + const endpointInts = verb ? this.resolveVerbEndpointInts(verb) : undefined + // Execute atomically with transaction system await this.transactionManager.executeTransaction(async (tx) => { // Operation 1: Remove from graph index - if (verb) { + if (verb && endpointInts) { tx.addOperation( - new RemoveFromGraphIndexOperation(this.graphIndex, verb as any) + new RemoveFromGraphIndexOperation( + this.graphIndex, verb, endpointInts.sourceInt, endpointInts.targetInt + ) ) } @@ -2158,6 +2323,10 @@ export class Brainy implements BrainyInterface { createdAt: existingAny.createdAt } + // 8.0 BigInt boundary: endpoints are unchanged across a type swap, so one + // resolution serves both the remove (rollback re-add) and the re-add. + const reindexInts = typeChanged ? this.resolveVerbEndpointInts(verbForIndex) : undefined + await this.transactionManager.executeTransaction(async (tx) => { tx.addOperation( new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata) @@ -2165,12 +2334,17 @@ export class Brainy implements BrainyInterface { // If the verb type changed, re-index in graph adjacency so traversal-by-type // stays consistent. The id is preserved across the swap. - if (typeChanged) { + if (typeChanged && reindexInts) { tx.addOperation( - new RemoveFromGraphIndexOperation(this.graphIndex, existing as any) + new RemoveFromGraphIndexOperation( + this.graphIndex, existing, reindexInts.sourceInt, reindexInts.targetInt + ) ) tx.addOperation( - new AddToGraphIndexOperation(this.graphIndex, verbForIndex) + new AddToGraphIndexOperation( + this.graphIndex, verbForIndex, reindexInts.sourceInt, reindexInts.targetInt, + (verbInt) => this.cacheVerbInt(verbInt, params.id) + ) ) } }) @@ -3027,10 +3201,11 @@ export class Brainy implements BrainyInterface { k ) - // Convert int IDs to UUIDs and paginate + // Convert int IDs (BigInt at the provider boundary) to UUIDs and + // paginate. Number() narrowing is lossless under the u32 guard. const idMapper = this.metadataIndex.getIdMapper() const allUuids = sortedIntIds - .map(intId => idMapper.getUuid(intId)) + .map(intId => idMapper.getUuid(Number(intId))) .filter((uuid): uuid is string => uuid !== undefined) const pageIds = allUuids.slice(offset, offset + limit) @@ -3759,8 +3934,9 @@ export class Brainy implements BrainyInterface { ) for (const verb of allVerbs) { + const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) tx.addOperation( - new RemoveFromGraphIndexOperation(this.graphIndex, verb as any) + new RemoveFromGraphIndexOperation(this.graphIndex, verb, sourceInt, targetInt) ) tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) @@ -4218,6 +4394,10 @@ export class Brainy implements BrainyInterface { } // getGraphIndex() will rebuild automatically if data exists (via _initializeGraphIndex) + // 8.0 u64 contract: the clone has its own metadata index (and idMapper), + // so thread the clone's resolver into its graph index + storage. + clone.wireGraphIdResolver() + // Mark as initialized clone.initialized = true clone.dimensions = this.dimensions @@ -4318,6 +4498,10 @@ export class Brainy implements BrainyInterface { this.graphIndex = await (this.storage as any).getGraphIndex() } + // 8.0 u64 contract: the branch switch recreated the metadata index (and + // idMapper) plus the graph index — re-thread the shared resolver. + this.wireGraphIdResolver() + // Reset lazy loading state when switching branches // Indexes contain data from previous branch, must rebuild for new branch this.lazyRebuildCompleted = false @@ -5697,11 +5881,18 @@ export class Brainy implements BrainyInterface { */ getTripleIntelligence(): TripleIntelligenceSystem { if (!this._tripleIntelligence) { - // Use core components directly - no lazy loading needed + // Use core components directly - no lazy loading needed. + // TripleIntelligenceSystem speaks UUIDs; adapt the BigInt graph-index + // contract through the coordinator's UUID-level helper so the int + // conversion stays at this boundary. this._tripleIntelligence = new TripleIntelligenceSystem( this.metadataIndex, this.index, - this.graphIndex, + { + getNeighbors: (id: string, direction?: 'in' | 'out' | 'both') => + this.getNeighborUuids(id, { direction }), + size: () => this.graphIndex.size() + }, async (text: string) => this.embedder(text), this.storage ) @@ -7917,13 +8108,20 @@ export class Brainy implements BrainyInterface { verbTypes?: Set, limit?: number ): Promise { - const neighbors = await this.graphIndex.getNeighbors(nodeId, { direction: graphDirection, limit }) + // 8.0 BigInt boundary: unmapped node → no relations. + const nodeInt = this.graphEntityInt(nodeId) + if (nodeInt === undefined) return [] + + const neighbors = this.entityIntsToUuids( + await this.graphIndex.getNeighbors(nodeInt, { direction: graphDirection, limit }) + ) if (!verbTypes || verbTypes.size === 0) return neighbors // Gather candidate edges in the relevant direction(s). - const verbIds: string[] = [] - if (graphDirection !== 'in') verbIds.push(...await this.graphIndex.getVerbIdsBySource(nodeId)) - if (graphDirection !== 'out') verbIds.push(...await this.graphIndex.getVerbIdsByTarget(nodeId)) + const verbInts: bigint[] = [] + if (graphDirection !== 'in') verbInts.push(...await this.graphIndex.getVerbIdsBySource(nodeInt)) + if (graphDirection !== 'out') verbInts.push(...await this.graphIndex.getVerbIdsByTarget(nodeInt)) + const verbIds = await this.resolveVerbIntsToIds(verbInts) const verbs = await this.graphIndex.getVerbsBatchCached(verbIds) const neighborSet = new Set(neighbors) @@ -8397,14 +8595,52 @@ export class Brainy implements BrainyInterface { if (subtypeFilter !== undefined) { // Multi-hop BFS with per-hop (verbType, subtype) predicate. Works on the - // open-core JS path at any depth. If a graph-index provider exposes a + // open-core JS path at any depth. When a graph-index provider exposes a // faster `findConnectedSubtype` (native BFS with the same semantics), - // we'll route through it transparently — same pattern as every other - // provider hook. + // `nativeSubtypeBfs` routes through it transparently — same pattern as + // every other provider hook. const subtypeArr = Array.isArray(subtypeFilter) ? subtypeFilter : [subtypeFilter] const subtypeSet = new Set(subtypeArr) + // Native fast path (D.3): single verb type + single subtype + outgoing + // walk match the native BFS semantics exactly (out-edges only, source + // excluded, visited-set cycle guard). Entity ints in, entity ints out — + // UUID conversion stays at this boundary. Returns null when the query + // shape (or provider) can't take the native route. + const nativeSubtypeBfs = async ( + anchor: string, + walk: 'in' | 'out' | 'both' + ): Promise | null> => { + if (walk !== 'out') return null + if (via === undefined || Array.isArray(via)) return null + if (subtypeArr.length !== 1) return null + const provider = this.graphIndex as Partial<{ + findConnectedSubtype( + sourceInt: bigint, + verbTypeIndex: number, + subtype: string | null, + depth: number, + limit?: number | null + ): Promise + }> + if (typeof provider.findConnectedSubtype !== 'function') return null + + const anchorInt = this.graphEntityInt(anchor) + if (anchorInt === undefined) return new Set() // unmapped → no relations + + const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType) + // No limit: match the JS BFS exactly — overall result limiting happens + // downstream against existingResults. + const reachedInts = await provider.findConnectedSubtype( + anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null + ) + return new Set(this.entityIntsToUuids(reachedInts)) + } + const bfsWithSubtype = async (anchor: string, walk: 'in' | 'out' | 'both'): Promise> => { + const nativeResult = await nativeSubtypeBfs(anchor, walk) + if (nativeResult !== null) return nativeResult + const visited = new Set([anchor]) const reached = new Set() let frontier = new Set([anchor]) @@ -8730,13 +8966,13 @@ export class Brainy implements BrainyInterface { if (constraints.to) { // Check if this entity connects TO the target (O(1) lookup) - const outgoingNeighbors = await this.graphIndex.getNeighbors(result.id, 'out') + const outgoingNeighbors = await this.getNeighborUuids(result.id, { direction: 'out' }) hasConnection = outgoingNeighbors.includes(constraints.to) } - + if (constraints.from && !hasConnection) { // Check if this entity connects FROM the source (O(1) lookup) - const incomingNeighbors = await this.graphIndex.getNeighbors(result.id, 'in') + const incomingNeighbors = await this.getNeighborUuids(result.id, { direction: 'in' }) hasConnection = incomingNeighbors.includes(constraints.from) } @@ -9368,6 +9604,24 @@ export class Brainy implements BrainyInterface { await this.lazyRebuildPromise } + /** + * @description Wire the shared UUID ↔ int resolver (the metadata index's + * idMapper) into the JS graph index and the storage layer (8.0 u64 + * contract). Called after the graph index is resolved on init, fork, and + * checkout so every consumer of the BigInt boundary shares one int + * universe. Native graph-index providers carry their own mapper and don't + * expose `setEntityIdMapper` — the storage-level wiring still applies so + * its verb read paths can convert UUIDs before provider calls. + */ + private wireGraphIdResolver(): void { + const resolver = this.metadataIndex.getIdMapper() + this.storage.setGraphEntityIdResolver(resolver) + const jsGraphIndex = this.graphIndex as Partial> + if (typeof jsGraphIndex.setEntityIdMapper === 'function') { + jsGraphIndex.setEntityIdMapper(resolver) + } + } + /** * @description Wire the HNSW connections codec (2.4.0 #3). Activates when * BOTH (a) the `graph:compression` provider is registered (cortex registers diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 2f6feb40..cb94560e 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -420,7 +420,7 @@ export interface HNSWVerbWithMetadata { * `relate()` / `getRelations()` surface, and storage adapters all speak it. */ export interface GraphVerb { - id: string // Unique identifier for the verb + id: string // Unique identifier — always a UUID (8.0 contract: brainy generates every verb id, so providers may key verb-int interning on the raw UUID bytes) sourceId: string // ID of the source noun targetId: string // ID of the target noun vector: Vector // Vector representation of the relationship @@ -439,6 +439,22 @@ export interface GraphVerb { createdAt?: number | { seconds: number; nanoseconds: number } // When the verb was created updatedAt?: number | { seconds: number; nanoseconds: number } // When the verb was last updated createdBy?: { augmentation: string; version: string } // Information about what created this verb + + /** + * Derived state: the source entity's interned integer ID (u64-safe BigInt). + * Populated by the coordinator (`brainy.ts`) from the shared entity-id mapper + * at add time, immediately before the verb is handed to the graph-index + * provider. NEVER persisted to storage JSON — the mapper is the source of + * truth, and persisting would denormalize state that can go stale across a + * mapper rebuild. + */ + sourceInt?: bigint + /** + * Derived state: the target entity's interned integer ID (u64-safe BigInt). + * Same lifecycle as {@link GraphVerb.sourceInt}: coordinator-populated at add + * time, never persisted to storage JSON. + */ + targetInt?: bigint } /** diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index a3a9faea..28efd2f9 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1,11 +1,20 @@ /** - * GraphAdjacencyIndex - Billion-Scale Graph Traversal Engine + * @module graph/graphAdjacencyIndex + * @description GraphAdjacencyIndex — billion-scale graph traversal engine. * - * NOW SCALES TO BILLIONS: LSM-tree storage reduces memory from 500GB to 1.3GB - * for 1 billion relationships while maintaining sub-5ms neighbor lookups. + * LSM-tree storage reduces memory from 500GB to 1.3GB for 1 billion + * relationships while maintaining sub-5ms neighbor lookups. * - * NO FALLBACKS - NO MOCKS - REAL PRODUCTION CODE - * Handles billions of relationships with sustainable memory usage + * **8.0 u64 boundary:** this class implements the BigInt + * {@link GraphIndexProvider} contract — entity ints in, entity/verb ints out. + * Internally everything stays string-keyed (UUID-keyed LSM trees) and u32: + * entity-int params resolve to UUIDs via the shared entity-id mapper + * (`getUuid(Number(big))` — lossless under the `EntityIdSpaceExceeded` u32 + * guard), and returns convert back with `BigInt(getOrAssign(uuid))`. Verb ints + * come from a small in-process interning map (see + * {@link GraphAdjacencyIndex.verbIntsToIds}); the interning is derived state, + * never persisted — `rebuild()` / the verb-id-set recovery path re-derive it + * from storage, which the JS index already does on cold start. */ import { GraphVerb, StorageAdapter } from '../coreTypes.js' @@ -21,6 +30,22 @@ export interface GraphIndexConfig { flushInterval?: number // Default: 30000ms } +/** + * @description The minimal UUID ↔ int resolver surface the JS graph index + * needs at its BigInt boundary. Satisfied by `EntityIdMapper` (and by whatever + * `MetadataIndexManager.getIdMapper()` / a native metadata index returns) — + * declared structurally here so the graph layer doesn't import the metadata + * layer. + */ +export interface GraphEntityIdResolver { + /** Resolve a UUID to its int, assigning a new one if absent (write path). */ + getOrAssign(uuid: string): number + /** Resolve a UUID to its int without assigning (read path). */ + getInt(uuid: string): number | undefined + /** Reverse-resolve an int to its UUID (`undefined` = unknown/deleted). */ + getUuid(intId: number): string | undefined +} + export interface GraphIndexStats { totalRelationships: number sourceNodes: number @@ -51,6 +76,18 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { // Now: Set stores only IDs (~100KB @ 1B verbs) = 1,280,000x reduction private verbIdSet = new Set() + // Verb-id interning for the BigInt boundary (8.0 u64 contract). + // Process-lifetime derived state: assigned on addVerb / rebuild / + // verb-id-set recovery, NEVER persisted. removeVerb keeps the entry so a + // verb int stays stable (and resolvable) for the index's lifetime. + private verbIdToInt = new Map() + private verbIntToId: string[] = [] + + // Shared UUID ↔ int resolver for entity ints at the BigInt boundary. + // Threaded in by the coordinator (brainy.ts) from the metadata index's + // idMapper — see setEntityIdMapper(). + private entityIdMapper?: GraphEntityIdResolver + // Infrastructure integration private storage: StorageAdapter private unifiedCache: UnifiedCache @@ -75,8 +112,13 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { return this.initialized } - constructor(storage: StorageAdapter, config: GraphIndexConfig = {}) { + constructor( + storage: StorageAdapter, + config: GraphIndexConfig = {}, + entityIdMapper?: GraphEntityIdResolver + ) { this.storage = storage + this.entityIdMapper = entityIdMapper this.config = { maxIndexSize: config.maxIndexSize ?? 100000, rebuildThreshold: config.rebuildThreshold ?? 0.1, @@ -116,6 +158,49 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { prodLog.info('GraphAdjacencyIndex initialized with LSM-tree storage (4 LSM-trees total)') } + /** + * @description Thread in the shared UUID ↔ int resolver used for entity-int + * conversion at the BigInt boundary. The coordinator (`brainy.ts`) calls this + * with `metadataIndex.getIdMapper()` right after index construction (init, + * fork, and checkout paths) so reads/writes share one int universe with the + * metadata index. Idempotent; safe to call again after a branch switch. + * @param mapper - The shared entity-id resolver. + * @returns Nothing. + */ + setEntityIdMapper(mapper: GraphEntityIdResolver): void { + this.entityIdMapper = mapper + } + + /** + * Resolve the entity-id mapper or fail loudly. The BigInt read methods are + * meaningless without a shared int universe — a missing mapper is a wiring + * bug, not a recoverable condition. + */ + private requireEntityIdMapper(): GraphEntityIdResolver { + if (!this.entityIdMapper) { + throw new Error( + 'GraphAdjacencyIndex: entityIdMapper not wired. The coordinator must ' + + 'call setEntityIdMapper(metadataIndex.getIdMapper()) (or pass it to ' + + 'the constructor) before BigInt-boundary reads.' + ) + } + return this.entityIdMapper + } + + /** + * Intern a verb-id string, assigning the next sequential u32 on first sight. + * Append-only for the index's lifetime — removeVerb keeps the entry so verb + * ints handed to callers stay resolvable. + */ + private internVerbId(verbId: string): number { + const existing = this.verbIdToInt.get(verbId) + if (existing !== undefined) return existing + const next = this.verbIntToId.length + this.verbIdToInt.set(verbId, next) + this.verbIntToId.push(verbId) + return next + } + /** * Initialize the graph index (lazy initialization) * Added defensive auto-rebuild check for verbIdSet consistency @@ -172,6 +257,10 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { for (const verb of result.items) { this.verbIdSet.add(verb.id) + // Re-derive the verb-int interning (process-lifetime state, never + // persisted — this recovery path is one of the two cold-start sources, + // alongside rebuild()). + this.internVerbId(verb.id) // Also update counts const verbType = verb.verb || 'unknown' this.relationshipCountsByType.set( @@ -190,49 +279,60 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } /** - * Core API - Neighbor lookup with LSM-tree storage + * @description Core API — neighbor lookup with LSM-tree storage (BigInt + * boundary). O(log n) with bloom filter optimization (90% of queries skip + * disk I/O); pagination support for high-degree nodes. The entity int is + * resolved to a UUID via the shared mapper (unknown int → empty result) and + * neighbor UUIDs convert back via `BigInt(getOrAssign(uuid))`. * - * O(log n) with bloom filter optimization (90% of queries skip disk I/O) - * Added pagination support for high-degree nodes - * - * @param id Entity ID to get neighbors for - * @param optionsOrDirection Optional: direction string OR options object - * @returns Array of neighbor IDs (paginated if limit/offset specified) + * @param id - Entity int to get neighbors for (from the shared idMapper). + * @param options - Optional direction ('both' default) + limit/offset. + * @returns Neighbor entity ints (paginated if limit/offset specified). * * @example - * // Get all neighbors (backward compatible) - * const all = await graphIndex.getNeighbors(id) + * // Get all neighbors of an entity int + * const all = await graphIndex.getNeighbors(42n) * * @example - * // Get outgoing neighbors (backward compatible) - * const out = await graphIndex.getNeighbors(id, 'out') - * - * @example - * // Get first 50 outgoing neighbors (new API) - * const page1 = await graphIndex.getNeighbors(id, { direction: 'out', limit: 50 }) - * - * @example - * // Paginate through neighbors - * const page1 = await graphIndex.getNeighbors(id, { limit: 100, offset: 0 }) - * const page2 = await graphIndex.getNeighbors(id, { limit: 100, offset: 100 }) + * // Get first 50 outgoing neighbors + * const page1 = await graphIndex.getNeighbors(42n, { direction: 'out', limit: 50 }) */ async getNeighbors( - id: string, - optionsOrDirection?: { + id: bigint, + options?: { direction?: 'in' | 'out' | 'both' limit?: number offset?: number - } | 'in' | 'out' | 'both' - ): Promise { + } + ): Promise { await this.ensureInitialized() - // Normalize old API (direction string) to new API (options object) - const options = typeof optionsOrDirection === 'string' - ? { direction: optionsOrDirection } - : (optionsOrDirection || {}) + const mapper = this.requireEntityIdMapper() + const uuid = mapper.getUuid(Number(id)) + if (uuid === undefined) { + // Unknown/deleted entity int — no edges by definition. + return [] + } + const neighborUuids = await this.getNeighborUuids(uuid, options) + return neighborUuids.map(neighborUuid => BigInt(mapper.getOrAssign(neighborUuid))) + } + + /** + * String-keyed neighbor lookup — the internal implementation behind + * {@link getNeighbors}. Kept UUID-based because the LSM trees are keyed by + * UUID; only the public contract speaks BigInt. + */ + private async getNeighborUuids( + id: string, + options?: { + direction?: 'in' | 'out' | 'both' + limit?: number + offset?: number + } + ): Promise { const startTime = performance.now() - const direction = options.direction || 'both' + const direction = options?.direction || 'both' const neighbors = new Set() // Query LSM-trees with bloom filter optimization @@ -255,8 +355,8 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { // Apply pagination if requested if (options?.limit !== undefined || options?.offset !== undefined) { - const offset = options.offset || 0 - const limit = options.limit !== undefined ? options.limit : result.length + const offset = options?.offset || 0 + const limit = options?.limit !== undefined ? options.limit : result.length result = result.slice(offset, offset + limit) } @@ -271,40 +371,34 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } /** - * Get verb IDs by source - Billion-scale optimization for getVerbsBySource + * @description Verb ints for all edges originating at `sourceInt` (BigInt + * boundary). O(log n) LSM-tree lookup with bloom filter optimization; + * filters out deleted verb IDs (tombstone deletion workaround); pagination + * support for entities with many relationships. Unknown entity int → empty + * result. Resolve returned ints back to verb-id strings with + * {@link verbIntsToIds}. * - * O(log n) LSM-tree lookup with bloom filter optimization - * Filters out deleted verb IDs (tombstone deletion workaround) - * Added pagination support for entities with many relationships - * - * @param sourceId Source entity ID - * @param options Optional configuration - * @param options.limit Maximum number of verb IDs to return (default: all) - * @param options.offset Number of verb IDs to skip (default: 0) - * @returns Array of verb IDs originating from this source (excluding deleted, paginated if requested) + * @param sourceInt - Source entity int (from the shared idMapper). + * @param options - Optional limit/offset pagination. + * @returns Verb ints originating from this source (excluding deleted). * * @example - * // Get all verb IDs (backward compatible) - * const all = await graphIndex.getVerbIdsBySource(sourceId) - * - * @example - * // Get first 50 verb IDs - * const page1 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 50 }) - * - * @example - * // Paginate through verb IDs - * const page1 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 100, offset: 0 }) - * const page2 = await graphIndex.getVerbIdsBySource(sourceId, { limit: 100, offset: 100 }) + * const verbInts = await graphIndex.getVerbIdsBySource(42n, { limit: 50 }) + * const verbIds = await graphIndex.verbIntsToIds(verbInts) */ async getVerbIdsBySource( - sourceId: string, + sourceInt: bigint, options?: { limit?: number offset?: number } - ): Promise { + ): Promise { await this.ensureInitialized() + const mapper = this.requireEntityIdMapper() + const sourceId = mapper.getUuid(Number(sourceInt)) + if (sourceId === undefined) return [] + const startTime = performance.now() const verbIds = await this.lsmTreeVerbsBySource.get(sourceId) const elapsed = performance.now() - startTime @@ -314,56 +408,37 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { prodLog.warn(`GraphAdjacencyIndex: Slow getVerbIdsBySource for ${sourceId}: ${elapsed.toFixed(2)}ms`) } - // Filter out deleted verb IDs (tombstone deletion workaround) - // LSM-tree retains all IDs, but verbIdSet tracks deletions - const allIds = verbIds || [] - let result = allIds.filter(id => this.verbIdSet.has(id)) - - // Apply pagination if requested - if (options?.limit !== undefined || options?.offset !== undefined) { - const offset = options.offset || 0 - const limit = options.limit !== undefined ? options.limit : result.length - result = result.slice(offset, offset + limit) - } - - return result + return this.verbIdsToPaginatedInts(verbIds || [], options) } /** - * Get verb IDs by target - Billion-scale optimization for getVerbsByTarget + * @description Verb ints for all edges pointing at `targetInt` (BigInt + * boundary). O(log n) LSM-tree lookup with bloom filter optimization; + * filters out deleted verb IDs (tombstone deletion workaround); pagination + * support for popular target entities. Unknown entity int → empty result. + * Resolve returned ints back to verb-id strings with {@link verbIntsToIds}. * - * O(log n) LSM-tree lookup with bloom filter optimization - * Filters out deleted verb IDs (tombstone deletion workaround) - * Added pagination support for popular target entities - * - * @param targetId Target entity ID - * @param options Optional configuration - * @param options.limit Maximum number of verb IDs to return (default: all) - * @param options.offset Number of verb IDs to skip (default: 0) - * @returns Array of verb IDs pointing to this target (excluding deleted, paginated if requested) + * @param targetInt - Target entity int (from the shared idMapper). + * @param options - Optional limit/offset pagination. + * @returns Verb ints pointing to this target (excluding deleted). * * @example - * // Get all verb IDs (backward compatible) - * const all = await graphIndex.getVerbIdsByTarget(targetId) - * - * @example - * // Get first 50 verb IDs - * const page1 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 50 }) - * - * @example - * // Paginate through verb IDs - * const page1 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 100, offset: 0 }) - * const page2 = await graphIndex.getVerbIdsByTarget(targetId, { limit: 100, offset: 100 }) + * const verbInts = await graphIndex.getVerbIdsByTarget(42n, { limit: 50 }) + * const verbIds = await graphIndex.verbIntsToIds(verbInts) */ async getVerbIdsByTarget( - targetId: string, + targetInt: bigint, options?: { limit?: number offset?: number } - ): Promise { + ): Promise { await this.ensureInitialized() + const mapper = this.requireEntityIdMapper() + const targetId = mapper.getUuid(Number(targetInt)) + if (targetId === undefined) return [] + const startTime = performance.now() const verbIds = await this.lsmTreeVerbsByTarget.get(targetId) const elapsed = performance.now() - startTime @@ -373,19 +448,42 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { prodLog.warn(`GraphAdjacencyIndex: Slow getVerbIdsByTarget for ${targetId}: ${elapsed.toFixed(2)}ms`) } - // Filter out deleted verb IDs (tombstone deletion workaround) - // LSM-tree retains all IDs, but verbIdSet tracks deletions - const allIds = verbIds || [] + return this.verbIdsToPaginatedInts(verbIds || [], options) + } + + /** + * Shared tail for the verb-int read methods: drop tombstoned ids (LSM trees + * retain all ids; verbIdSet tracks deletions), apply pagination, and intern + * the survivors to verb ints. Interning on the read path is safe — ids are + * assigned deterministically within the process lifetime and never persisted. + */ + private verbIdsToPaginatedInts( + allIds: string[], + options?: { limit?: number; offset?: number } + ): bigint[] { let result = allIds.filter(id => this.verbIdSet.has(id)) // Apply pagination if requested if (options?.limit !== undefined || options?.offset !== undefined) { - const offset = options.offset || 0 - const limit = options.limit !== undefined ? options.limit : result.length + const offset = options?.offset || 0 + const limit = options?.limit !== undefined ? options.limit : result.length result = result.slice(offset, offset + limit) } - return result + return result.map(id => BigInt(this.internVerbId(id))) + } + + /** + * @description Batch reverse resolver: verb ints → verb-id strings (the + * REQUIRED half of the 8.0 contract Brainy's warm cache feeds from). Reads + * the in-process interning map populated by `addVerb`, `rebuild()`, and the + * verb-id-set recovery path — the JS index derives the interning from + * storage on cold start, so no sidecar persistence exists or is needed. + * @param verbInts - Verb ints as returned by the verb-int read methods. + * @returns One entry per input, order-preserving; `null` for unknown ints. + */ + async verbIntsToIds(verbInts: bigint[]): Promise<(string | null)[]> { + return verbInts.map(verbInt => this.verbIntToId[Number(verbInt)] ?? null) } /** @@ -529,15 +627,32 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } /** - * Add relationship to index using LSM-tree storage + * @description Add a relationship to the index (BigInt boundary). The + * coordinator resolves both endpoint ints via `idMapper.getOrAssign` and + * mirrors them onto `verb.sourceInt`/`verb.targetInt` before calling. The + * JS index keys its LSM trees by the verb's endpoint UUIDs, so the int + * params carry no extra information here — they exist for contract parity + * with native providers whose trees are int-keyed. + * @param verb - The verb to index (endpoint UUIDs are authoritative). + * @param sourceInt - The source entity's interned int (contract parity). + * @param targetInt - The target entity's interned int (contract parity). + * @returns The interned verb int for `verb.id` (stable for the index lifetime). */ - async addVerb(verb: GraphVerb): Promise { + async addVerb(verb: GraphVerb, sourceInt: bigint, targetInt: bigint): Promise { await this.ensureInitialized() + return BigInt(await this.indexVerb(verb)) + } + /** + * String-keyed indexing core shared by {@link addVerb} and {@link rebuild}. + * Returns the interned verb int. + */ + private async indexVerb(verb: GraphVerb): Promise { const startTime = performance.now() // Track verb ID (memory-efficient: IDs only, full objects loaded on-demand via UnifiedCache) this.verbIdSet.add(verb.id) + const verbInt = this.internVerbId(verb.id) // Add to LSM-trees (outgoing and incoming edges) await this.lsmTreeSource.add(verb.sourceId, verb.targetId) @@ -561,12 +676,17 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { if (elapsed > 10.0) { prodLog.warn(`GraphAdjacencyIndex: Slow addVerb for ${verb.id}: ${elapsed.toFixed(2)}ms`) } + + return verbInt } /** - * Remove relationship from index - * Note: LSM-tree edges persist (tombstone deletion not yet implemented) - * Only removes from verb cache and updates counts + * @description Remove a relationship from the index by its id string. + * LSM-tree edges persist (tombstone deletion via verbIdSet filtering); the + * verb's interned int is intentionally retained so previously returned verb + * ints stay resolvable via {@link verbIntsToIds}. + * @param verbId - The verb's UUID string. + * @returns Resolves once the verb no longer appears in reads. */ async removeVerb(verbId: string): Promise { await this.ensureInitialized() @@ -624,6 +744,11 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { this.totalRelationshipsIndexed = 0 // CRITICAL FIX - Clear relationship counts to prevent accumulation this.relationshipCountsByType.clear() + // Re-derive verb-int interning from scratch — it's process-lifetime + // derived state (never persisted), so a rebuild starts a fresh + // generation of verb ints alongside the fresh verbIdSet. + this.verbIdToInt.clear() + this.verbIntToId = [] // Note: LSM-trees will be recreated from storage via their own initialization // Verb data will be loaded on-demand via UnifiedCache @@ -655,7 +780,7 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { confidence: verb.confidence, weight: verb.weight } - await this.addVerb(graphVerb) + await this.indexVerb(graphVerb) totalVerbs++ } diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index f290274f..e47af77e 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -58,12 +58,16 @@ interface HeapEntry { /** * Unified column store coordinator. * + * Entity ints cross the {@link ColumnStoreProvider} boundary as `bigint` + * (8.0 u64 contract); internally this JS baseline stays u32 — `Number(bigint)` + * narrowing is lossless under the `EntityIdSpaceExceeded` u32 guard. + * * @example * const store = new ColumnStore() * await store.init(storage, idMapper) - * store.addEntity(42, { createdAt: Date.now(), status: 'active' }) + * store.addEntity(42n, { createdAt: Date.now(), status: 'active' }) * await store.flush() - * const newest = await store.sortTopK('createdAt', 'desc', 10) + * const newest = await store.sortTopK('createdAt', 'desc', 10) // bigint[] */ export class ColumnStore implements ColumnStoreProvider { private storage!: StorageAdapter @@ -170,8 +174,13 @@ export class ColumnStore implements ColumnStoreProvider { * one entry per element is added. * * Auto-flushes when any tail buffer exceeds its threshold. + * + * @param entityIntId - The entity's interned integer ID (u64-safe BigInt; + * narrowed to u32 internally under the `EntityIdSpaceExceeded` guard) + * @param fields - Map of field name → value (or array of values for multi-value) */ - addEntity(entityIntId: number, fields: Record): void { + addEntity(entityIntId: bigint, fields: Record): void { + const intId = Number(entityIntId) for (const [field, rawValue] of Object.entries(fields)) { if (rawValue === undefined || rawValue === null) continue @@ -179,11 +188,11 @@ export class ColumnStore implements ColumnStoreProvider { if (Array.isArray(rawValue)) { for (const v of rawValue) { if (v !== undefined && v !== null) { - this.pushToBuffer(field, v, entityIntId, true) + this.pushToBuffer(field, v, intId, true) } } } else { - this.pushToBuffer(field, rawValue, entityIntId, false) + this.pushToBuffer(field, rawValue, intId, false) } } } @@ -195,11 +204,16 @@ export class ColumnStore implements ColumnStoreProvider { * during ALL queries) and removes any pending entries from tail buffers. * The global bitmap is persisted in the manifest on flush and cleared * during compaction. + * + * @param entityIntId - The entity's interned integer ID (u64-safe BigInt; + * narrowed to u32 internally under the `EntityIdSpaceExceeded` guard) */ - removeEntity(entityIntId: number): void { + removeEntity(entityIntId: bigint): void { + const intId = Number(entityIntId) + // Remove from tail buffers (pending writes) for (const [, buffer] of this.tailBuffers) { - buffer.remove(entityIntId) + buffer.remove(intId) } // Add to global deleted bitmap for every known field @@ -209,7 +223,7 @@ export class ColumnStore implements ColumnStoreProvider { deleted = new RoaringBitmap32() this.deletedEntities.set(field, deleted) } - deleted.add(entityIntId) + deleted.add(intId) } } @@ -273,26 +287,28 @@ export class ColumnStore implements ColumnStoreProvider { } /** - * Sort top-K: return K entity int IDs in sorted order. + * Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt). * * Uses k-way merge across all segment cursors + tail buffer via a min/max heap. * Complexity: O(K log S) where S = number of cursors. */ - async sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise { - return this.mergeSort(field, order, k, null) + async sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise { + const intIds = await this.mergeSort(field, order, k, null) + return intIds.map((id) => BigInt(id)) } /** - * Filtered sort top-K: return K entity int IDs in sorted order, - * considering only entities present in the filter bitmap. + * Filtered sort top-K: return K entity int IDs in sorted order (u64-safe + * BigInt), considering only entities present in the filter bitmap. */ async filteredSortTopK( filterBitmap: RoaringBitmap32, field: string, order: 'asc' | 'desc', k: number - ): Promise { - return this.mergeSort(field, order, k, filterBitmap) + ): Promise { + const intIds = await this.mergeSort(field, order, k, filterBitmap) + return intIds.map((id) => BigInt(id)) } /** diff --git a/src/indexes/columnStore/types.ts b/src/indexes/columnStore/types.ts index 116ac172..58e1c4d6 100644 --- a/src/indexes/columnStore/types.ts +++ b/src/indexes/columnStore/types.ts @@ -170,13 +170,19 @@ export const FLAG_MULTI_VALUE = 0x01 * Brainy ships a TypeScript baseline that implements this interface. * Cortex registers a Rust-accelerated implementation at higher priority. * The MetadataIndex coordinator calls whichever is registered. + * + * **8.0 u64 contract — BigInt entity ints at the boundary.** Entity ints flow + * in and out as `bigint` so a native u64-keyed column store loses nothing at + * the wrapper. Brainy's JS baseline stays u32 internally (`Number(bigint)` + * narrowing is lossless under the shipped `EntityIdSpaceExceeded` u32 guard) + * and its filter/range bitmaps stay Roaring32 — only this boundary widens. */ export interface ColumnStoreProvider { /** * Initialize the column store: discover fields, load manifests. * * @param storage - The storage adapter for reading/writing segment files - * @param idMapper - Shared EntityIdMapper for UUID ↔ u32 conversion + * @param idMapper - Shared EntityIdMapper for UUID ↔ int conversion */ init(storage: StorageAdapter, idMapper: EntityIdMapper): Promise @@ -184,17 +190,17 @@ export interface ColumnStoreProvider { * Index an entity's field values. For multi-value fields (arrays), * one entry per element is added to the column. * - * @param entityIntId - The entity's u32 integer ID (from EntityIdMapper) + * @param entityIntId - The entity's interned integer ID (u64-safe BigInt) * @param fields - Map of field name → value (or array of values for multi-value) */ - addEntity(entityIntId: number, fields: Record): void + addEntity(entityIntId: bigint, fields: Record): void /** * Remove an entity from all indexed columns by adding tombstones. * - * @param entityIntId - The entity's u32 integer ID + * @param entityIntId - The entity's interned integer ID (u64-safe BigInt) */ - removeEntity(entityIntId: number): void + removeEntity(entityIntId: bigint): void /** * Point filter: find all entities where field equals value. @@ -222,9 +228,9 @@ export interface ColumnStoreProvider { * @param field - Field to sort by * @param order - 'asc' or 'desc' * @param k - Maximum number of results - * @returns Sorted array of entity int IDs + * @returns Sorted array of entity int IDs (u64-safe BigInt) */ - sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise + sortTopK(field: string, order: 'asc' | 'desc', k: number): Promise /** * Filtered sort top-K: same as sortTopK but only considers entities @@ -234,14 +240,14 @@ export interface ColumnStoreProvider { * @param field - Field to sort by * @param order - 'asc' or 'desc' * @param k - Maximum number of results - * @returns Sorted array of entity int IDs (subset of filterBitmap) + * @returns Sorted array of entity int IDs (u64-safe BigInt, subset of filterBitmap) */ filteredSortTopK( filterBitmap: RoaringBitmap32, field: string, order: 'asc' | 'desc', k: number - ): Promise + ): Promise /** * Get all distinct values for a field across all segments. diff --git a/src/plugin.ts b/src/plugin.ts index 89a57d0f..07aa5ac7 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -150,8 +150,20 @@ export interface MetadataIndexProvider { getStats(): Promise - /** The shared UUID ↔ int mapper. Brainy reads `.getUuid(intId)` off the result. */ - getIdMapper(): { getUuid(intId: number): string | undefined } + /** + * The shared UUID ↔ int mapper — the single source of truth for entity-int + * resolution at the provider boundary. The coordinator (`brainy.ts`) resolves + * UUID → int exactly once before every graph-index call (`getOrAssign` on + * writes, `getInt` on reads — `undefined` means "never mapped", i.e. the + * entity has no relations) and converts provider-returned ints back with + * `getUuid`. Ints are u32 today (the `EntityIdSpaceExceeded` guard enforces + * the ceiling on the JS path), so `Number(bigint)` narrowing is lossless. + */ + getIdMapper(): { + getOrAssign(uuid: string): number + getInt(uuid: string): number | undefined + getUuid(intId: number): string | undefined + } /** The column store the coordinator delegates `where`/`orderBy` to. */ readonly columnStore: import('./indexes/columnStore/types.js').ColumnStoreProvider @@ -160,19 +172,75 @@ export interface MetadataIndexProvider { /** * The `'graphIndex'` provider — a drop-in for `GraphAdjacencyIndex`. * Brainy calls this surface via `this.graphIndex.*` (some optional-chained). + * + * **8.0 u64 contract — BigInt at the boundary.** Reads take entity ints + * (from the metadata index's idMapper) and return entity/verb ints as + * `bigint[]`. The coordinator owns ALL UUID ↔ int conversion: it resolves + * UUIDs to ints once at the `brainy.ts` boundary (`getOrAssign` on writes, + * `getInt` on reads, returning empty results for unmapped UUIDs without + * calling the provider) and maps returned ints back (`getUuid` for entities, + * {@link GraphIndexProvider.verbIntsToIds} for verbs). Implementations may + * stay u32 internally — `Number(bigint)` narrowing is lossless under the + * shipped `EntityIdSpaceExceeded` u32 guard — but must speak the bigint + * contract at this surface. */ export interface GraphIndexProvider { /** `false` until the index has loaded; Brainy probes this before fast paths. */ readonly isInitialized: boolean + /** + * @description Entity ints reachable from `id` (1 hop), deduped. + * @param id - The entity's interned int (from the shared idMapper). + * @param options - Direction (`'both'` default) and limit/offset pagination. + * @returns Neighbor entity ints. Empty when the entity has no edges. + */ getNeighbors( - id: string, - optionsOrDirection?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number } | 'in' | 'out' | 'both' - ): Promise - getVerbIdsBySource(sourceId: string, options?: { limit?: number; offset?: number }): Promise - getVerbIdsByTarget(targetId: string, options?: { limit?: number; offset?: number }): Promise + id: bigint, + options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number } + ): Promise + /** + * @description Verb ints for all edges originating at `sourceInt`. + * @param sourceInt - The source entity's interned int. + * @param options - Optional limit/offset pagination. + * @returns Verb ints, resolvable via {@link GraphIndexProvider.verbIntsToIds}. + */ + getVerbIdsBySource(sourceInt: bigint, options?: { limit?: number; offset?: number }): Promise + /** + * @description Verb ints for all edges pointing at `targetInt`. + * @param targetInt - The target entity's interned int. + * @param options - Optional limit/offset pagination. + * @returns Verb ints, resolvable via {@link GraphIndexProvider.verbIntsToIds}. + */ + getVerbIdsByTarget(targetInt: bigint, options?: { limit?: number; offset?: number }): Promise + /** + * @description Batch reverse resolver: verb ints → verb-id strings. REQUIRED — + * the provider owns the durable verb-int interning (Brainy keeps only a + * bounded in-memory warm cache fed by `addVerb` returns and this resolver; + * pure optimization, no durability role). + * @param verbInts - Verb ints as returned by the read methods. + * @returns One entry per input, order-preserving; `null` for unknown ints. + */ + verbIntsToIds(verbInts: bigint[]): Promise<(string | null)[]> getVerbsBatchCached(verbIds: string[]): Promise> + /** + * @description Index one verb. The coordinator resolves both endpoint ints + * via `idMapper.getOrAssign` and mirrors them onto `verb.sourceInt` / + * `verb.targetInt` before the call. + * @param verb - The verb to index (endpoint UUIDs + derived `sourceInt`/`targetInt`). + * @param sourceInt - The source entity's interned int. + * @param targetInt - The target entity's interned int. + * @returns The interned verb int for `verb.id` (feeds Brainy's warm cache). + */ + addVerb(verb: GraphVerb, sourceInt: bigint, targetInt: bigint): Promise + /** + * @description Remove one verb from the index by its id string. The verb's + * interned int stays reserved (ints are never recycled within a generation). + * @param verbId - The verb's UUID string. + * @returns Resolves once the verb no longer appears in reads. + */ + removeVerb(verbId: string): Promise + rebuild(): Promise flush(): Promise close(): Promise diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 34a7d855..0a72c15b 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -4,6 +4,7 @@ */ import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js' +import type { GraphEntityIdResolver } from '../graph/graphAdjacencyIndex.js' import { GraphVerb, @@ -176,6 +177,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected isInitialized = false protected graphIndex?: GraphAdjacencyIndex protected graphIndexPromise?: Promise + /** + * Shared UUID ↔ int resolver for the graph index's BigInt boundary. + * Wired by Brainy via {@link setGraphEntityIdResolver}; until then the verb + * read paths fall back to shard iteration. + */ + protected graphEntityIdResolver?: GraphEntityIdResolver protected readOnly = false // Write-through cache for read-after-write consistency @@ -442,6 +449,26 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.graphIndexPromise = Promise.resolve(index) } + /** + * @description Wire the shared UUID ↔ int resolver used at the graph index's + * BigInt boundary (8.0 u64 contract). Brainy calls this with + * `metadataIndex.getIdMapper()` after the graph index is resolved (init, + * fork, and checkout paths). The storage layer needs it to convert UUIDs to + * entity ints before `getVerbIdsBySource`/`getVerbIdsByTarget` calls and to + * resolve returned verb ints back to verb-id strings. Until it's wired, the + * verb read paths fall back to shard iteration (correct, just slower). + * @param resolver - The shared entity-id resolver. + * @returns Nothing. + */ + public setGraphEntityIdResolver(resolver: GraphEntityIdResolver): void { + this.graphEntityIdResolver = resolver + // Thread the resolver into the JS graph index too, when present — native + // providers carry their own mapper and don't expose this setter. + if (this.graphIndex && typeof (this.graphIndex as GraphAdjacencyIndex).setEntityIdMapper === 'function') { + (this.graphIndex as GraphAdjacencyIndex).setEntityIdMapper(resolver) + } + } + /** * Ensure the storage adapter is initialized */ @@ -2107,7 +2134,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ private async _initializeGraphIndex(): Promise { prodLog.info('Initializing GraphAdjacencyIndex...') - this.graphIndex = new GraphAdjacencyIndex(this) + // Thread the shared entity-id resolver when already wired (re-init after + // invalidateGraphIndex); on first init Brainy wires it right after. + this.graphIndex = new GraphAdjacencyIndex(this, {}, this.graphEntityIdResolver) // Check if we need to rebuild from existing data const sampleVerbs = await this.getVerbs({ pagination: { limit: 1 } }) @@ -3610,10 +3639,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`) - // Fast path - use GraphAdjacencyIndex if available (lazy-loaded) - if (this.graphIndex && this.graphIndex.isInitialized) { + // Fast path - use GraphAdjacencyIndex if available (lazy-loaded). + // 8.0 BigInt boundary: convert the UUID to an entity int up front and + // resolve returned verb ints back to verb-id strings. + if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) { try { - const verbIds = await this.graphIndex.getVerbIdsBySource(sourceId) + const sourceInt = this.graphEntityIdResolver.getInt(sourceId) + if (sourceInt === undefined) { + // Never-mapped UUID — the entity has no relations by definition. + return [] + } + const verbInts = await this.graphIndex.getVerbIdsBySource(BigInt(sourceInt)) + const verbIds = (await this.graphIndex.verbIntsToIds(verbInts)) + .filter((id): id is string => id !== null) prodLog.debug(`[BaseStorage] GraphAdjacencyIndex found ${verbIds.length} verb IDs for sourceId=${sourceId}`) // PERFORMANCE FIX - Batch fetch verbs + metadata (eliminates N+1 pattern) @@ -3860,10 +3898,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { ): Promise { await this.ensureInitialized() - // Fast path - use GraphAdjacencyIndex if available (lazy-loaded) - if (this.graphIndex && this.graphIndex.isInitialized) { + // Fast path - use GraphAdjacencyIndex if available (lazy-loaded). + // 8.0 BigInt boundary: convert the UUID to an entity int up front and + // resolve returned verb ints back to verb-id strings. + if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) { try { - const verbIds = await this.graphIndex.getVerbIdsByTarget(targetId) + const targetInt = this.graphEntityIdResolver.getInt(targetId) + if (targetInt === undefined) { + // Never-mapped UUID — the entity has no relations by definition. + return [] + } + const verbInts = await this.graphIndex.getVerbIdsByTarget(BigInt(targetInt)) + const verbIds = (await this.graphIndex.verbIntsToIds(verbInts)) + .filter((id): id is string => id !== null) const results: HNSWVerbWithMetadata[] = [] for (const verbId of verbIds) { diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 932c93ec..00aed0d2 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -11,7 +11,7 @@ import type { JsHnswVectorIndex } from '../../hnsw/hnswIndex.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js' -import type { GraphAdjacencyIndex } from '../../graph/graphAdjacencyIndex.js' +import type { GraphIndexProvider } from '../../plugin.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' @@ -152,18 +152,35 @@ export class RemoveFromMetadataIndexOperation implements Operation { * * Rollback strategy: * - Remove verb from graph index + * + * 8.0 u64 contract: the coordinator resolves both endpoint ints via the + * shared idMapper (`getOrAssign`) and passes them alongside the verb; the + * provider returns the interned verb int, which is surfaced through the + * optional `onVerbInt` callback so the coordinator can feed its warm cache. */ export class AddToGraphIndexOperation implements Operation { readonly name = 'AddToGraphIndex' + /** + * @param index - The graph-index provider (JS baseline or native). + * @param verb - The verb to index (`sourceInt`/`targetInt` mirrored on it). + * @param sourceInt - The source entity's interned int. + * @param targetInt - The target entity's interned int. + * @param onVerbInt - Optional hook invoked with the interned verb int + * returned by the provider (feeds the coordinator's verb-int warm cache). + */ constructor( - private readonly index: GraphAdjacencyIndex, - private readonly verb: GraphVerb + private readonly index: GraphIndexProvider, + private readonly verb: GraphVerb, + private readonly sourceInt: bigint, + private readonly targetInt: bigint, + private readonly onVerbInt?: (verbInt: bigint) => void ) {} async execute(): Promise { // Add verb to graph index - await this.index.addVerb(this.verb) + const verbInt = await this.index.addVerb(this.verb, this.sourceInt, this.targetInt) + this.onVerbInt?.(verbInt) // Return rollback action return async () => { @@ -178,13 +195,25 @@ export class AddToGraphIndexOperation implements Operation { * * Rollback strategy: * - Re-add verb to graph index + * + * 8.0 u64 contract: rollback re-adds through `addVerb(verb, sourceInt, + * targetInt)`, so the coordinator resolves the endpoint ints up front + * (while the entity → int mappings are guaranteed to still exist). */ export class RemoveFromGraphIndexOperation implements Operation { readonly name = 'RemoveFromGraphIndex' + /** + * @param index - The graph-index provider (JS baseline or native). + * @param verb - The verb being removed (required for rollback re-add). + * @param sourceInt - The source entity's interned int (rollback re-add). + * @param targetInt - The target entity's interned int (rollback re-add). + */ constructor( - private readonly index: GraphAdjacencyIndex, - private readonly verb: GraphVerb // Required for rollback + private readonly index: GraphIndexProvider, + private readonly verb: GraphVerb, // Required for rollback + private readonly sourceInt: bigint, + private readonly targetInt: bigint ) {} async execute(): Promise { @@ -194,7 +223,7 @@ export class RemoveFromGraphIndexOperation implements Operation { // Return rollback action return async () => { // Re-add verb with original data - await this.index.addVerb(this.verb) + await this.index.addVerb(this.verb, this.sourceInt, this.targetInt) } } } diff --git a/src/types/graphTypes.ts b/src/types/graphTypes.ts index 57c2afdb..6e456a02 100644 --- a/src/types/graphTypes.ts +++ b/src/types/graphTypes.ts @@ -368,7 +368,7 @@ export interface GraphNoun { * Represents relationships between nouns */ export interface GraphVerb { - id: string // Unique identifier for the verb + id: string // Unique identifier — always a UUID (8.0 contract: brainy generates every verb id, so providers may key verb-int interning on the raw UUID bytes) sourceId: string // Entity UUID of the source noun targetId: string // Entity UUID of the target noun label?: string // Optional descriptive label @@ -381,6 +381,21 @@ export interface GraphVerb { embedding?: number[] // Vector representation of the relationship confidence?: number // Confidence score (0-1) weight?: number // Strength/importance of the relationship + + /** + * Derived state: the source entity's interned integer ID (u64-safe BigInt). + * Populated by the coordinator from the shared entity-id mapper at add time, + * immediately before the verb is handed to the graph-index provider. NEVER + * persisted to storage JSON — the mapper is the source of truth, and + * persisting would denormalize state that can go stale across a mapper rebuild. + */ + sourceInt?: bigint + /** + * Derived state: the target entity's interned integer ID (u64-safe BigInt). + * Same lifecycle as {@link GraphVerb.sourceInt}: coordinator-populated at add + * time, never persisted to storage JSON. + */ + targetInt?: bigint } /** diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index e71975b4..b910e7bd 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1412,7 +1412,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { fieldsMap[field] = value } } - this.columnStore.addEntity(entityIntId, fieldsMap) + this.columnStore.addEntity(BigInt(entityIntId), fieldsMap) } // Adaptive auto-flush based on usage patterns @@ -1501,7 +1501,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { if (this.columnStore) { const intId = this.idMapper.getInt(id) if (intId !== undefined) { - this.columnStore.removeEntity(intId) + this.columnStore.removeEntity(BigInt(intId)) } } @@ -2073,7 +2073,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { if (hasFilter && filteredIds.length === 0) return [] - let sortedIntIds: number[] + let sortedIntIds: bigint[] if (hasFilter) { // Build filter bitmap for the column store const filterBitmap = new RoaringBitmap32() @@ -2091,9 +2091,10 @@ export class MetadataIndexManager implements MetadataIndexProvider { ) } - // Convert int IDs back to UUIDs + // Convert int IDs back to UUIDs. Number() narrowing is lossless — the + // shipped EntityIdSpaceExceeded guard caps the JS mapper at u32. return sortedIntIds - .map(intId => this.idMapper.getUuid(intId)) + .map(intId => this.idMapper.getUuid(Number(intId))) .filter((uuid): uuid is string => uuid !== undefined) } diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 17748340..c3246d96 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -524,6 +524,21 @@ export function validateUpdateParams(params: UpdateParams): void { * Validate relate parameters */ export function validateRelateParams(params: RelateParams): void { + // 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy. + // RelateParams has no `id` field — an untyped caller passing one would + // previously have it silently ignored (a generated UUID was used instead). + // Teach instead of surprise: explain the contract and how ids actually work. + const suppliedId = (params as RelateParams & { id?: unknown }).id + if (suppliedId !== undefined) { + throw new Error( + `relate() does not accept a custom id (got: ${JSON.stringify(suppliedId)}). ` + + 'Verb ids are UUIDs by contract in 8.0 — brainy generates one for every ' + + 'relationship and returns it from relate(). Graph-index providers key ' + + 'verb-int interning on the raw UUID bytes, so custom verb ids are not ' + + 'supported. Remove the id field and use the returned id instead.' + ) + } + // Universal truths if (!params.from) { throw new Error('from entity ID is required') diff --git a/tests/performance/graph-scale-performance.test.ts b/tests/performance/graph-scale-performance.test.ts index 91143fd3..9aacbfd0 100644 --- a/tests/performance/graph-scale-performance.test.ts +++ b/tests/performance/graph-scale-performance.test.ts @@ -15,6 +15,7 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js' +import { EntityIdMapper } from '../../src/utils/entityIdMapper.js' import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' import { performance } from 'perf_hooks' @@ -112,8 +113,18 @@ describe('🧠 Graph Scale Performance Benchmarks', () => { let brain: Brainy let graphIndex: GraphAdjacencyIndex let storage: MemoryStorage + let idMapper: EntityIdMapper const scale = getTestScale() + /** Resolve a UUID to its entity int for the 8.0 BigInt boundary. */ + const entityInt = (uuid: string): bigint => BigInt(idMapper.getOrAssign(uuid)) + + /** Map returned entity ints back to UUIDs. */ + const intsToUuids = (ints: bigint[]): string[] => + ints + .map((i) => idMapper.getUuid(Number(i))) + .filter((u): u is string => u !== undefined) + // Performance tracking const lookupStats = new PerformanceStats() const updateStats = new PerformanceStats() @@ -130,10 +141,13 @@ describe('🧠 Graph Scale Performance Benchmarks', () => { storage = new MemoryStorage() await storage.init() + idMapper = new EntityIdMapper({ storage }) + await idMapper.init() + graphIndex = new GraphAdjacencyIndex(storage, { maxIndexSize: scale.nodes, autoOptimize: true - }) + }, idMapper) // Initialize Brainy for unified testing brain = new Brainy({ requireSubtype: false, @@ -226,7 +240,7 @@ describe('🧠 Graph Scale Performance Benchmarks', () => { // Warm up the index await graphIndex.rebuild() - await graphIndex.getNeighbors('node-100') // Warm up + await graphIndex.getNeighbors(entityInt('node-100')) // Warm up // Test random lookups const testIterations = Math.min(1000, scale.nodes / 10) @@ -236,7 +250,7 @@ describe('🧠 Graph Scale Performance Benchmarks', () => { for (const nodeId of sampleNodes) { const startTime = performance.now() - const neighbors = await graphIndex.getNeighbors(nodeId) + const neighbors = await graphIndex.getNeighbors(entityInt(nodeId)) const elapsed = performance.now() - startTime lookupStats.addSample(elapsed) @@ -408,9 +422,9 @@ describe('🧠 Graph Scale Performance Benchmarks', () => { visited.add(id) nodesTraversed++ - // Get neighbors - const neighbors = await graphIndex.getNeighbors(id, 'out') - for (const neighbor of neighbors) { + // Get neighbors (BigInt boundary: ints out, mapped back to UUIDs) + const neighborInts = await graphIndex.getNeighbors(entityInt(id), { direction: 'out' }) + for (const neighbor of intsToUuids(neighborInts)) { if (!visited.has(neighbor)) { queue.push({ id: neighbor, depth: depth + 1 }) } @@ -687,7 +701,7 @@ describe('🧠 Graph Scale Performance Benchmarks', () => { switch (operationType) { case 0: // Neighbor lookup - operations.push(graphIndex.getNeighbors(`node-${Math.floor(Math.random() * scale.nodes)}`)) + operations.push(graphIndex.getNeighbors(entityInt(`node-${Math.floor(Math.random() * scale.nodes)}`))) break case 1: // Unified find operations.push(brain.find({ diff --git a/tests/unit/brainy/relate-duplicate-optimization.test.ts b/tests/unit/brainy/relate-duplicate-optimization.test.ts index 67070e47..dfd4da88 100644 --- a/tests/unit/brainy/relate-duplicate-optimization.test.ts +++ b/tests/unit/brainy/relate-duplicate-optimization.test.ts @@ -151,8 +151,12 @@ describe('Duplicate Check Optimization', () => { type: VerbType.RelatesTo }) - // Access GraphIndex to ensure verb is cached - const verbIds = await (brain as any).graphIndex.getVerbIdsBySource(entityA) + // Access GraphIndex to ensure verb is cached (8.0 BigInt boundary: + // UUID → entity int in, verb ints out, resolved back via verbIntsToIds) + const idMapper = (brain as any).metadataIndex.getIdMapper() + const sourceInt = BigInt(idMapper.getInt(entityA)!) + const verbInts: bigint[] = await (brain as any).graphIndex.getVerbIdsBySource(sourceInt) + const verbIds = await (brain as any).graphIndex.verbIntsToIds(verbInts) expect(verbIds).toContain(relationId1) // Attempt duplicate (should use cached verb) diff --git a/tests/unit/graph/bigint-contract.test.ts b/tests/unit/graph/bigint-contract.test.ts new file mode 100644 index 00000000..77d3356f --- /dev/null +++ b/tests/unit/graph/bigint-contract.test.ts @@ -0,0 +1,230 @@ +/** + * @module bigint-contract.test + * @description 8.0 u64 provider contract — BigInt boundary of the JS + * GraphAdjacencyIndex and the coordinator that drives it. + * + * Covers: + * - bigint round-trip through the JS graph index: `addVerb` returns a stable + * verb int, `getVerbIdsBySource`/`getVerbIdsByTarget` return that int, and + * `verbIntsToIds` maps it back to the verb-id string (null for unknown ints) + * - `getNeighbors` speaks entity ints both ways via the shared idMapper + * - unmapped entity ints / UUIDs produce empty reads, never errors + * - a missing idMapper fails loudly (wiring bug, not a silent fallback) + * - coordinator-level behavior: relate → getRelations/neighbors → unrelate + * works end-to-end over the BigInt boundary (public API unchanged) + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { GraphAdjacencyIndex } from '../../../src/graph/graphAdjacencyIndex.js' +import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { NounType, VerbType } from '../../../src/types/graphTypes.js' +import type { GraphVerb } from '../../../src/coreTypes.js' + +// Verb ids are UUID-shaped by contract in 8.0 (storage enforces the shape on +// every save; cortex keys verb-int interning on the raw UUID bytes). +const VERB_UUID_1 = '11111111-1111-4111-8111-111111111111' +const VERB_UUID_2 = '22222222-2222-4222-8222-222222222222' +const VERB_UUID_3 = '33333333-3333-4333-8333-333333333333' + +/** Build a minimal GraphVerb for direct index-level tests. */ +function makeVerb(id: string, sourceId: string, targetId: string): GraphVerb { + return { + id, + sourceId, + targetId, + vector: [], + type: VerbType.RelatedTo, + verb: VerbType.RelatedTo + } +} + +describe('GraphAdjacencyIndex — BigInt boundary (JS implementation)', () => { + let storage: MemoryStorage + let idMapper: EntityIdMapper + let index: GraphAdjacencyIndex + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + idMapper = new EntityIdMapper({ storage, storageKey: 'test:graph:idMapper' }) + await idMapper.init() + index = new GraphAdjacencyIndex(storage, {}, idMapper) + }) + + afterEach(async () => { + await index.close() + }) + + it('addVerb returns a stable verb int; reads return it; verbIntsToIds maps back', async () => { + const sourceInt = BigInt(idMapper.getOrAssign('uuid-source')) + const targetInt = BigInt(idMapper.getOrAssign('uuid-target')) + const verb = makeVerb(VERB_UUID_1, 'uuid-source', 'uuid-target') + verb.sourceInt = sourceInt + verb.targetInt = targetInt + + const verbInt = await index.addVerb(verb, sourceInt, targetInt) + expect(typeof verbInt).toBe('bigint') + + // Re-adding the same verb id returns the SAME interned int (stable). + const verbIntAgain = await index.addVerb(verb, sourceInt, targetInt) + expect(verbIntAgain).toBe(verbInt) + + // Both directed verb-int reads surface the int from addVerb. + const bySource = await index.getVerbIdsBySource(sourceInt) + expect(bySource).toContain(verbInt) + + const byTarget = await index.getVerbIdsByTarget(targetInt) + expect(byTarget).toContain(verbInt) + + // Batch reverse resolution is order-preserving with null for unknowns. + const resolved = await index.verbIntsToIds([verbInt, 999_999n]) + expect(resolved).toEqual([VERB_UUID_1, null]) + }) + + it('getNeighbors speaks entity ints in both directions', async () => { + const aInt = BigInt(idMapper.getOrAssign('uuid-a')) + const bInt = BigInt(idMapper.getOrAssign('uuid-b')) + const verb = makeVerb(VERB_UUID_2, 'uuid-a', 'uuid-b') + + await index.addVerb(verb, aInt, bInt) + + const out = await index.getNeighbors(aInt, { direction: 'out' }) + expect(out).toEqual([bInt]) + + const incoming = await index.getNeighbors(bInt, { direction: 'in' }) + expect(incoming).toEqual([aInt]) + + const both = await index.getNeighbors(aInt) + expect(both).toContain(bInt) + }) + + it('reads for unmapped entity ints return empty without error', async () => { + // 424242 was never assigned by the mapper — no UUID behind it. + const ghost = 424_242n + expect(await index.getNeighbors(ghost)).toEqual([]) + expect(await index.getVerbIdsBySource(ghost)).toEqual([]) + expect(await index.getVerbIdsByTarget(ghost)).toEqual([]) + }) + + it('verb ints stay resolvable after removeVerb (interning is append-only)', async () => { + const sInt = BigInt(idMapper.getOrAssign('uuid-s')) + const tInt = BigInt(idMapper.getOrAssign('uuid-t')) + const verb = makeVerb(VERB_UUID_3, 'uuid-s', 'uuid-t') + + const verbInt = await index.addVerb(verb, sInt, tInt) + // Persist the verb (vector + metadata — getVerb requires both) so + // removeVerb can resolve its type for count updates. + await storage.saveVerb({ + id: verb.id, + vector: [], + connections: new Map>(), + verb: VerbType.RelatedTo, + sourceId: verb.sourceId, + targetId: verb.targetId + }) + await storage.saveVerbMetadata(verb.id, { + verb: VerbType.RelatedTo, + createdAt: Date.now() + }) + + await index.removeVerb(verb.id) + + // Removed verb no longer appears in reads… + expect(await index.getVerbIdsBySource(sInt)).toEqual([]) + // …but its int still reverse-resolves (append-only interning). + expect(await index.verbIntsToIds([verbInt])).toEqual([VERB_UUID_3]) + }) + + it('fails loudly when the entityIdMapper is not wired', async () => { + const bare = new GraphAdjacencyIndex(storage) + await expect(bare.getNeighbors(1n)).rejects.toThrow(/entityIdMapper not wired/) + await bare.close() + }) +}) + +describe('Coordinator — relate/related end-to-end over the BigInt boundary', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('relate → getRelations → neighbors round-trips (public API unchanged)', async () => { + const personId = await brain.add({ + data: { name: 'Ada' }, + type: NounType.Person + }) + const projectId = await brain.add({ + data: { name: 'Analytical Engine' }, + type: NounType.Thing + }) + + const relId = await brain.relate({ + from: personId, + to: projectId, + type: VerbType.WorksWith + }) + + // getRelations resolves verb ints back to verb-id strings internally. + const relations = await brain.getRelations({ from: personId }) + expect(relations).toHaveLength(1) + expect(relations[0].id).toBe(relId) + expect(relations[0].to).toBe(projectId) + + // neighbors() routes through getNeighbors(bigint) + int → UUID mapping. + const out = await brain.neighbors(personId, { direction: 'outgoing' }) + expect(out).toContain(projectId) + + const incoming = await brain.neighbors(projectId, { direction: 'incoming' }) + expect(incoming).toContain(personId) + }) + + it('relate feeds the verb-int warm cache; duplicate detection still works', async () => { + const a = await brain.add({ data: { name: 'A' }, type: NounType.Thing }) + const b = await brain.add({ data: { name: 'B' }, type: NounType.Thing }) + + const first = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + + // The AddToGraphIndexOperation callback recorded the verb int → id pair. + const warmCache: Map = (brain as any).verbIntWarmCache + expect([...warmCache.values()]).toContain(first) + + // Duplicate check runs through getVerbIdsBySource(bigint) + resolution. + const duplicate = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + expect(duplicate).toBe(first) + }) + + it('unrelate removes the edge from BigInt-boundary reads', async () => { + const a = await brain.add({ data: { name: 'A' }, type: NounType.Thing }) + const b = await brain.add({ data: { name: 'B' }, type: NounType.Thing }) + + const relId = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + await brain.unrelate(relId) + + const relations = await brain.getRelations({ from: a }) + expect(relations).toHaveLength(0) + }) + + it('neighbors() of an unknown UUID returns empty without error', async () => { + const result = await brain.neighbors('00000000-0000-4000-8000-000000000000') + expect(result).toEqual([]) + }) + + it('relate() rejects a caller-supplied verb id with a teaching error (8.0 contract)', async () => { + const a = await brain.add({ data: { name: 'A' }, type: NounType.Thing }) + const b = await brain.add({ data: { name: 'B' }, type: NounType.Thing }) + + // RelateParams has no `id` field — untyped callers passing one used to be + // silently ignored. 8.0 teaches: verb ids are brainy-generated UUIDs. + await expect( + brain.relate({ from: a, to: b, type: VerbType.RelatedTo, id: 'my-custom-id' } as any) + ).rejects.toThrow(/Verb ids are UUIDs by contract in 8\.0/) + }) +}) diff --git a/tests/unit/graph/graphIndex-pagination.test.ts b/tests/unit/graph/graphIndex-pagination.test.ts index 633ae616..ac9c7695 100644 --- a/tests/unit/graph/graphIndex-pagination.test.ts +++ b/tests/unit/graph/graphIndex-pagination.test.ts @@ -1,12 +1,14 @@ /** - * GraphAdjacencyIndex Pagination Tests (v5.8.0) + * GraphAdjacencyIndex Pagination Tests (v5.8.0, updated for the 8.0 u64 contract) * * Tests for pagination support in GraphIndex methods: * - getNeighbors() with limit/offset * - getVerbIdsBySource() with limit/offset * - getVerbIdsByTarget() with limit/offset * - * Verifies backward compatibility and new pagination features + * 8.0 BigInt boundary: entity ints in (resolved via the metadata index's + * idMapper), entity/verb ints out (`bigint[]`). Entity ints map back to UUIDs + * via `idMapper.getUuid(Number(int))`; verb ints via `verbIntsToIds()`. */ import { describe, it, expect, beforeEach } from 'vitest' @@ -18,6 +20,25 @@ describe('GraphAdjacencyIndex Pagination', () => { let centralId: string let neighborIds: string[] + /** The brain's graph index (BigInt provider contract). */ + const graphIndex = () => (brain as any).graphIndex + + /** The shared UUID ↔ int resolver. */ + const idMapper = () => (brain as any).metadataIndex.getIdMapper() + + /** Resolve a UUID to its entity int (must already be mapped). */ + const entityInt = (uuid: string): bigint => { + const intId = idMapper().getInt(uuid) + expect(intId).toBeDefined() + return BigInt(intId!) + } + + /** Map returned entity ints back to UUIDs. */ + const intsToUuids = (ints: bigint[]): string[] => + ints + .map((i) => idMapper().getUuid(Number(i))) + .filter((u: string | undefined): u is string => u !== undefined) + beforeEach(async () => { brain = new Brainy({ requireSubtype: false }) await brain.init() @@ -47,50 +68,42 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('getNeighbors() Pagination', () => { - it('should return all neighbors without pagination (backward compatible)', async () => { - const graphIndex = (brain as any).graphIndex - - const neighbors = await graphIndex.getNeighbors(centralId) + it('should return all neighbors without pagination', async () => { + const neighborInts = await graphIndex().getNeighbors(entityInt(centralId)) + const neighbors = intsToUuids(neighborInts) // Should return all 50 neighbors expect(neighbors).toHaveLength(50) expect(neighbors.every((id: string) => neighborIds.includes(id))).toBe(true) }) - it('should return all outgoing neighbors with direction parameter (backward compatible)', async () => { - const graphIndex = (brain as any).graphIndex - - // Old API: direction as string - const neighbors = await graphIndex.getNeighbors(centralId, 'out') + it('should return all outgoing neighbors with direction option', async () => { + const neighbors = await graphIndex().getNeighbors(entityInt(centralId), { direction: 'out' }) expect(neighbors).toHaveLength(50) }) it('should paginate with limit only', async () => { - const graphIndex = (brain as any).graphIndex - - const page1 = await graphIndex.getNeighbors(centralId, { limit: 10 }) + const page1Ints = await graphIndex().getNeighbors(entityInt(centralId), { limit: 10 }) + const page1 = intsToUuids(page1Ints) expect(page1).toHaveLength(10) expect(page1.every((id: string) => neighborIds.includes(id))).toBe(true) }) it('should paginate with offset only', async () => { - const graphIndex = (brain as any).graphIndex - - const all = await graphIndex.getNeighbors(centralId) - const offsetResults = await graphIndex.getNeighbors(centralId, { offset: 10 }) + const all = await graphIndex().getNeighbors(entityInt(centralId)) + const offsetResults = await graphIndex().getNeighbors(entityInt(centralId), { offset: 10 }) // Offset 10 should return all after first 10 expect(offsetResults).toHaveLength(all.length - 10) }) it('should paginate with both limit and offset', async () => { - const graphIndex = (brain as any).graphIndex - - const page1 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 0 }) - const page2 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 10 }) - const page3 = await graphIndex.getNeighbors(centralId, { limit: 10, offset: 20 }) + const central = entityInt(centralId) + const page1: bigint[] = await graphIndex().getNeighbors(central, { limit: 10, offset: 0 }) + const page2: bigint[] = await graphIndex().getNeighbors(central, { limit: 10, offset: 10 }) + const page3: bigint[] = await graphIndex().getNeighbors(central, { limit: 10, offset: 20 }) // Each page should have 10 results expect(page1).toHaveLength(10) @@ -98,39 +111,32 @@ describe('GraphAdjacencyIndex Pagination', () => { expect(page3).toHaveLength(10) // Pages should not overlap - const page1Set = new Set(page1) const page2Set = new Set(page2) const page3Set = new Set(page3) - const overlap12 = page1.filter(id => page2Set.has(id)) - const overlap23 = page2.filter(id => page3Set.has(id)) + const overlap12 = page1.filter((id) => page2Set.has(id)) + const overlap23 = page2.filter((id) => page3Set.has(id)) expect(overlap12).toHaveLength(0) expect(overlap23).toHaveLength(0) }) it('should handle offset beyond total count', async () => { - const graphIndex = (brain as any).graphIndex - - const results = await graphIndex.getNeighbors(centralId, { offset: 100 }) + const results = await graphIndex().getNeighbors(entityInt(centralId), { offset: 100 }) // Offset 100 > 50 total → empty array expect(results).toHaveLength(0) }) it('should handle limit = 0', async () => { - const graphIndex = (brain as any).graphIndex - - const results = await graphIndex.getNeighbors(centralId, { limit: 0 }) + const results = await graphIndex().getNeighbors(entityInt(centralId), { limit: 0 }) expect(results).toHaveLength(0) }) it('should paginate with direction filter', async () => { - const graphIndex = (brain as any).graphIndex - // Get first 10 outgoing neighbors - const page1 = await graphIndex.getNeighbors(centralId, { + const page1 = await graphIndex().getNeighbors(entityInt(centralId), { direction: 'out', limit: 10, offset: 0 @@ -140,8 +146,6 @@ describe('GraphAdjacencyIndex Pagination', () => { }) it('should handle pagination with incoming direction', async () => { - const graphIndex = (brain as any).graphIndex - // Create some incoming relationships const sourceId = await brain.add({ data: { name: 'Source' }, @@ -155,7 +159,7 @@ describe('GraphAdjacencyIndex Pagination', () => { }) // Get incoming neighbors with pagination - const incoming = await graphIndex.getNeighbors(centralId, { + const incoming = await graphIndex().getNeighbors(entityInt(centralId), { direction: 'in', limit: 10 }) @@ -165,38 +169,36 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('getVerbIdsBySource() Pagination', () => { - it('should return all verb IDs without pagination (backward compatible)', async () => { - const graphIndex = (brain as any).graphIndex + it('should return all verb ints without pagination and resolve them back to ids', async () => { + const verbInts: bigint[] = await graphIndex().getVerbIdsBySource(entityInt(centralId)) - const verbIds = await graphIndex.getVerbIdsBySource(centralId) + // Should return all 50 verb ints + expect(verbInts).toHaveLength(50) - // Should return all 50 verb IDs + // Every int must resolve back to a verb-id string + const verbIds: (string | null)[] = await graphIndex().verbIntsToIds(verbInts) expect(verbIds).toHaveLength(50) + expect(verbIds.every((id) => typeof id === 'string')).toBe(true) }) - it('should paginate verb IDs with limit', async () => { - const graphIndex = (brain as any).graphIndex - - const page1 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10 }) + it('should paginate verb ints with limit', async () => { + const page1 = await graphIndex().getVerbIdsBySource(entityInt(centralId), { limit: 10 }) expect(page1).toHaveLength(10) }) - it('should paginate verb IDs with offset', async () => { - const graphIndex = (brain as any).graphIndex - - const all = await graphIndex.getVerbIdsBySource(centralId) - const offsetResults = await graphIndex.getVerbIdsBySource(centralId, { offset: 10 }) + it('should paginate verb ints with offset', async () => { + const all = await graphIndex().getVerbIdsBySource(entityInt(centralId)) + const offsetResults = await graphIndex().getVerbIdsBySource(entityInt(centralId), { offset: 10 }) expect(offsetResults).toHaveLength(all.length - 10) }) - it('should paginate verb IDs with limit and offset', async () => { - const graphIndex = (brain as any).graphIndex - - const page1 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 0 }) - const page2 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 10 }) - const page3 = await graphIndex.getVerbIdsBySource(centralId, { limit: 10, offset: 20 }) + it('should paginate verb ints with limit and offset', async () => { + const central = entityInt(centralId) + const page1: bigint[] = await graphIndex().getVerbIdsBySource(central, { limit: 10, offset: 0 }) + const page2: bigint[] = await graphIndex().getVerbIdsBySource(central, { limit: 10, offset: 10 }) + const page3: bigint[] = await graphIndex().getVerbIdsBySource(central, { limit: 10, offset: 20 }) // Each page should have 10 results expect(page1).toHaveLength(10) @@ -209,34 +211,28 @@ describe('GraphAdjacencyIndex Pagination', () => { expect(unique.size).toBe(30) // All unique }) - it('should handle pagination edge cases for verb IDs', async () => { - const graphIndex = (brain as any).graphIndex - + it('should handle pagination edge cases for verb ints', async () => { // Offset beyond total - const beyondTotal = await graphIndex.getVerbIdsBySource(centralId, { offset: 100 }) + const beyondTotal = await graphIndex().getVerbIdsBySource(entityInt(centralId), { offset: 100 }) expect(beyondTotal).toHaveLength(0) // Limit = 0 - const zeroLimit = await graphIndex.getVerbIdsBySource(centralId, { limit: 0 }) + const zeroLimit = await graphIndex().getVerbIdsBySource(entityInt(centralId), { limit: 0 }) expect(zeroLimit).toHaveLength(0) }) }) describe('getVerbIdsByTarget() Pagination', () => { - it('should return all verb IDs targeting an entity (backward compatible)', async () => { - const graphIndex = (brain as any).graphIndex - + it('should return all verb ints targeting an entity', async () => { // Pick a neighbor that's a target of relationships const targetId = neighborIds[0] - const verbIds = await graphIndex.getVerbIdsByTarget(targetId) + const verbInts = await graphIndex().getVerbIdsByTarget(entityInt(targetId)) // Should have at least 1 (the relationship from central) - expect(verbIds.length).toBeGreaterThanOrEqual(1) + expect(verbInts.length).toBeGreaterThanOrEqual(1) }) - it('should paginate verb IDs by target with limit', async () => { - const graphIndex = (brain as any).graphIndex - + it('should paginate verb ints by target with limit', async () => { // Create entity with many incoming relationships const popularTarget = await brain.add({ data: { name: 'Popular Target' }, @@ -257,28 +253,29 @@ describe('GraphAdjacencyIndex Pagination', () => { } // Paginate - const page1 = await graphIndex.getVerbIdsByTarget(popularTarget, { limit: 10 }) - const page2 = await graphIndex.getVerbIdsByTarget(popularTarget, { limit: 10, offset: 10 }) + const target = entityInt(popularTarget) + const page1: bigint[] = await graphIndex().getVerbIdsByTarget(target, { limit: 10 }) + const page2: bigint[] = await graphIndex().getVerbIdsByTarget(target, { limit: 10, offset: 10 }) expect(page1).toHaveLength(10) expect(page2).toHaveLength(10) // No overlap - const overlap = page1.filter((id: string) => page2.includes(id)) + const overlap = page1.filter((id) => page2.includes(id)) expect(overlap).toHaveLength(0) }) }) describe('Performance with Pagination', () => { it('should maintain sub-5ms performance with pagination', async () => { - const graphIndex = (brain as any).graphIndex + const central = entityInt(centralId) // Multiple paginated queries should be fast const startTime = performance.now() - await graphIndex.getNeighbors(centralId, { limit: 10, offset: 0 }) - await graphIndex.getNeighbors(centralId, { limit: 10, offset: 10 }) - await graphIndex.getNeighbors(centralId, { limit: 10, offset: 20 }) + await graphIndex().getNeighbors(central, { limit: 10, offset: 0 }) + await graphIndex().getNeighbors(central, { limit: 10, offset: 10 }) + await graphIndex().getNeighbors(central, { limit: 10, offset: 20 }) const elapsed = performance.now() - startTime @@ -310,12 +307,11 @@ describe('GraphAdjacencyIndex Pagination', () => { }) } - const graphIndex = (brain as any).graphIndex - // Paginate in chunks of 25 - const chunks: string[][] = [] + const hubInt = entityInt(hub) + const chunks: bigint[][] = [] for (let offset = 0; offset < 100; offset += 25) { - const chunk = await graphIndex.getNeighbors(hub, { + const chunk = await graphIndex().getNeighbors(hubInt, { direction: 'out', limit: 25, offset diff --git a/tests/unit/indexes/columnStore/column-store-interchange.test.ts b/tests/unit/indexes/columnStore/column-store-interchange.test.ts index f79038ea..9de9a21c 100644 --- a/tests/unit/indexes/columnStore/column-store-interchange.test.ts +++ b/tests/unit/indexes/columnStore/column-store-interchange.test.ts @@ -44,7 +44,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => { it('flushBuffer writes segment bytes as a raw blob at the cortex-shared key', async () => { for (let i = 0; i < 5; i++) { - store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 }) + store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i * 10 }) } await store.flush() @@ -63,9 +63,9 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => { it('flushBuffer writes the DELETED bitmap as a raw blob too (not an envelope)', async () => { for (let i = 0; i < 3; i++) { - store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i }) + store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i }) } - store.removeEntity(idMapper.getInt('u-1')!) + store.removeEntity(BigInt(idMapper.getInt('u-1')!)) await store.flush() const rawDel = await (storage as any).loadBinaryBlob('_column_index/score/DELETED') @@ -135,7 +135,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => { const readStore = new ColumnStore() await readStore.init(storage, idMapper) const sortedInts = await readStore.sortTopK('score', 'asc', 10) - const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i))) expect(sortedUuids).toEqual(['legacy-a', 'legacy-b', 'legacy-c']) await readStore.close() }) @@ -153,15 +153,15 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => { // Add a couple of live entities so a manifest exists (init only loads // DELETED for fields with a manifest on disk). - store.addEntity(idMapper.getOrAssign('alive-1'), { score: 100 }) - store.addEntity(idMapper.getOrAssign('alive-2'), { score: 200 }) + store.addEntity(BigInt(idMapper.getOrAssign('alive-1')), { score: 100 }) + store.addEntity(BigInt(idMapper.getOrAssign('alive-2')), { score: 200 }) await store.flush() await store.close() const readStore = new ColumnStore() await readStore.init(storage, idMapper) const sortedInts = await readStore.sortTopK('score', 'asc', 10) - const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i))) // The dead entity must NOT appear; the alive ones must. expect(sortedUuids).not.toContain('dead') @@ -176,7 +176,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => { it('write (new format) → re-init → read returns the same data', async () => { for (let i = 0; i < 5; i++) { - store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 }) + store.addEntity(BigInt(idMapper.getOrAssign(`u-${i}`)), { score: i * 10 }) } await store.flush() await store.close() @@ -184,7 +184,7 @@ describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => { const readStore = new ColumnStore() await readStore.init(storage, idMapper) const sortedInts = await readStore.sortTopK('score', 'asc', 10) - const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(Number(i))) expect(sortedUuids).toEqual(['u-0', 'u-1', 'u-2', 'u-3', 'u-4']) await readStore.close() }) diff --git a/tests/unit/indexes/columnStore/column-store.test.ts b/tests/unit/indexes/columnStore/column-store.test.ts index 73ef14ed..b9404fa6 100644 --- a/tests/unit/indexes/columnStore/column-store.test.ts +++ b/tests/unit/indexes/columnStore/column-store.test.ts @@ -35,48 +35,48 @@ describe('ColumnStore', () => { describe('sortTopK', () => { it('returns entities sorted by numeric field ascending', async () => { - store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 }) - store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 }) - store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 }) + store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 }) + store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 }) await store.flush() const result = await store.sortTopK('createdAt', 'asc', 10) - const uuids = result.map(id => idMapper.getUuid(id)) + const uuids = result.map(id => idMapper.getUuid(Number(id))) expect(uuids).toEqual(['b', 'c', 'a']) }) it('returns entities sorted by numeric field descending', async () => { - store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 }) - store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 }) - store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 }) + store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 }) + store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 }) await store.flush() const result = await store.sortTopK('createdAt', 'desc', 10) - const uuids = result.map(id => idMapper.getUuid(id)) + const uuids = result.map(id => idMapper.getUuid(Number(id))) expect(uuids).toEqual(['a', 'c', 'b']) }) it('respects limit K', async () => { for (let i = 0; i < 50; i++) { - store.addEntity(idMapper.getOrAssign(`e${i}`), { createdAt: i * 1000 }) + store.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { createdAt: i * 1000 }) } await store.flush() const result = await store.sortTopK('createdAt', 'desc', 5) expect(result).toHaveLength(5) // Should be the 5 highest timestamps - const uuids = result.map(id => idMapper.getUuid(id)) + const uuids = result.map(id => idMapper.getUuid(Number(id))) expect(uuids).toEqual(['e49', 'e48', 'e47', 'e46', 'e45']) }) it('sorts string fields lexicographically', async () => { - store.addEntity(idMapper.getOrAssign('a'), { status: 'pending' }) - store.addEntity(idMapper.getOrAssign('b'), { status: 'active' }) - store.addEntity(idMapper.getOrAssign('c'), { status: 'inactive' }) + store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'pending' }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'active' }) + store.addEntity(BigInt(idMapper.getOrAssign('c')), { status: 'inactive' }) await store.flush() const result = await store.sortTopK('status', 'asc', 10) - const uuids = result.map(id => idMapper.getUuid(id)) + const uuids = result.map(id => idMapper.getUuid(Number(id))) expect(uuids).toEqual(['b', 'c', 'a']) // active, inactive, pending }) @@ -97,16 +97,16 @@ describe('ColumnStore', () => { const idC = idMapper.getOrAssign('c') const idD = idMapper.getOrAssign('d') - store.addEntity(idA, { createdAt: 400 }) - store.addEntity(idB, { createdAt: 100 }) - store.addEntity(idC, { createdAt: 300 }) - store.addEntity(idD, { createdAt: 200 }) + store.addEntity(BigInt(idA), { createdAt: 400 }) + store.addEntity(BigInt(idB), { createdAt: 100 }) + store.addEntity(BigInt(idC), { createdAt: 300 }) + store.addEntity(BigInt(idD), { createdAt: 200 }) await store.flush() // Filter: only B and C const bitmap = new RoaringBitmap32([idB, idC]) const result = await store.filteredSortTopK(bitmap, 'createdAt', 'desc', 10) - const uuids = result.map(id => idMapper.getUuid(id)) + const uuids = result.map(id => idMapper.getUuid(Number(id))) expect(uuids).toEqual(['c', 'b']) // 300, 100 }) }) @@ -121,9 +121,9 @@ describe('ColumnStore', () => { const idB = idMapper.getOrAssign('b') const idC = idMapper.getOrAssign('c') - store.addEntity(idA, { status: 'active' }) - store.addEntity(idB, { status: 'inactive' }) - store.addEntity(idC, { status: 'active' }) + store.addEntity(BigInt(idA), { status: 'active' }) + store.addEntity(BigInt(idB), { status: 'inactive' }) + store.addEntity(BigInt(idC), { status: 'active' }) await store.flush() const bitmap = await store.filter('status', 'active') @@ -133,7 +133,7 @@ describe('ColumnStore', () => { }) it('returns empty bitmap for no match', async () => { - store.addEntity(idMapper.getOrAssign('a'), { status: 'active' }) + store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' }) await store.flush() const bitmap = await store.filter('status', 'deleted') @@ -150,7 +150,7 @@ describe('ColumnStore', () => { const ids: number[] = [] for (let i = 0; i < 10; i++) { ids.push(idMapper.getOrAssign(`e${i}`)) - store.addEntity(ids[i], { price: i * 10 }) + store.addEntity(BigInt(ids[i]), { price: i * 10 }) } await store.flush() @@ -174,16 +174,16 @@ describe('ColumnStore', () => { const idB = idMapper.getOrAssign('b') const idC = idMapper.getOrAssign('c') - store.addEntity(idA, { createdAt: 100 }) - store.addEntity(idB, { createdAt: 200 }) - store.addEntity(idC, { createdAt: 300 }) + store.addEntity(BigInt(idA), { createdAt: 100 }) + store.addEntity(BigInt(idB), { createdAt: 200 }) + store.addEntity(BigInt(idC), { createdAt: 300 }) await store.flush() - store.removeEntity(idB) + store.removeEntity(BigInt(idB)) await store.flush() const result = await store.sortTopK('createdAt', 'asc', 10) - const uuids = result.map(id => idMapper.getUuid(id)) + const uuids = result.map(id => idMapper.getUuid(Number(id))) expect(uuids).toEqual(['a', 'c']) }) }) @@ -196,12 +196,12 @@ describe('ColumnStore', () => { it('sorts correctly across multiple L0 segments', async () => { // Flush threshold is 10, so 25 entities creates 3 segments for (let i = 0; i < 25; i++) { - store.addEntity(idMapper.getOrAssign(`e${i}`), { ts: i }) + store.addEntity(BigInt(idMapper.getOrAssign(`e${i}`)), { ts: i }) } await store.flush() const result = await store.sortTopK('ts', 'desc', 5) - const uuids = result.map(id => idMapper.getUuid(id)) + const uuids = result.map(id => idMapper.getUuid(Number(id))) expect(uuids).toEqual(['e24', 'e23', 'e22', 'e21', 'e20']) }) }) @@ -212,9 +212,9 @@ describe('ColumnStore', () => { describe('persistence', () => { it('survives close + reload', async () => { - store.addEntity(idMapper.getOrAssign('a'), { createdAt: 300 }) - store.addEntity(idMapper.getOrAssign('b'), { createdAt: 100 }) - store.addEntity(idMapper.getOrAssign('c'), { createdAt: 200 }) + store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 300 }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { createdAt: 100 }) + store.addEntity(BigInt(idMapper.getOrAssign('c')), { createdAt: 200 }) await store.flush() await store.close() @@ -223,7 +223,7 @@ describe('ColumnStore', () => { await store2.init(storage, idMapper) const result = await store2.sortTopK('createdAt', 'desc', 10) - const uuids = result.map(id => idMapper.getUuid(id)) + const uuids = result.map(id => idMapper.getUuid(Number(id))) expect(uuids).toEqual(['a', 'c', 'b']) await store2.close() @@ -239,8 +239,8 @@ describe('ColumnStore', () => { const idA = idMapper.getOrAssign('docA') const idB = idMapper.getOrAssign('docB') - store.addEntity(idA, { __words__: ['machine', 'learning', 'algorithm'] }) - store.addEntity(idB, { __words__: ['neural', 'network', 'machine'] }) + store.addEntity(BigInt(idA), { __words__: ['machine', 'learning', 'algorithm'] }) + store.addEntity(BigInt(idB), { __words__: ['neural', 'network', 'machine'] }) await store.flush() // Point filter: "machine" should match both @@ -267,10 +267,10 @@ describe('ColumnStore', () => { describe('getFilterValues', () => { it('returns distinct values sorted', async () => { - store.addEntity(idMapper.getOrAssign('a'), { status: 'pending' }) - store.addEntity(idMapper.getOrAssign('b'), { status: 'active' }) - store.addEntity(idMapper.getOrAssign('c'), { status: 'active' }) - store.addEntity(idMapper.getOrAssign('d'), { status: 'inactive' }) + store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'pending' }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'active' }) + store.addEntity(BigInt(idMapper.getOrAssign('c')), { status: 'active' }) + store.addEntity(BigInt(idMapper.getOrAssign('d')), { status: 'inactive' }) await store.flush() const values = await store.getFilterValues('status') @@ -288,7 +288,7 @@ describe('ColumnStore', () => { }) it('returns true after adding data', async () => { - store.addEntity(idMapper.getOrAssign('a'), { createdAt: 100 }) + store.addEntity(BigInt(idMapper.getOrAssign('a')), { createdAt: 100 }) expect(store.hasField('createdAt')).toBe(true) }) })