From a3d6fdb8b359a1f1eb45b6033e7fe0b034359bfb Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 21 Jun 2026 08:12:22 -0700 Subject: [PATCH] =?UTF-8?q?feat(8.0):=20GraphAccelerationProvider=20contra?= =?UTF-8?q?ct=20=E2=80=94=20the=20native=20graph-engine=20seam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defines the optional native graph-acceleration provider (cor 3.0) that brainy feature-detects and routes brain.graph.* / related({node}) / find({connected}) to, falling back to pure-TS adjacency when absent. This is the published seam cor wires its native build against (mirrors the existing VersionedIndexProvider pattern: defined in plugin.ts, exported from index.ts, implemented externally, feature-detected at runtime). Contract (locked cross-team in the design thread): - GraphAccelerationProvider: traverse, edgesForNode, graphCursorOpen/Next/Close, pageRank, connectedComponents, shortestPath, neighborhoodSample, topByDegree. Every read op is generation-aware (optional trailing generation?: bigint) so db.asOf(g).graph.* resolves historically; omitted = now. - Subgraph: columnar wire format (BigInt64Array node/edge columns, Uint16Array edgeTypes, optional Uint8Array nodeDepth, truncated flag) — parallel typed arrays, never array-of-objects; brainy maps u64<->UUID lazily for rendered rows. - OpaqueIdSet: a find() universe forwarded opaquely into traverse/search for the zero-crossing query->expand fusion (brainy never inspects it; the provider version-tags + validates the envelope). - VectorIndexProvider.search gains allowedIds?: OpaqueIdSet | ReadonlySet (predicate pushdown — recovers filtered recall lost to post-filtering). - EntityIdMapperProvider gains entityIntsToUuids(BigInt64Array) — the bigint batch reverse resolver for Subgraph.nodes; implemented on the JS mapper (TS fallback). All new public types exported from index.ts. Test: entityIntsToUuids round-trip + order-preservation + empty-int sentinel. --- src/index.ts | 22 ++ src/plugin.ts | 321 +++++++++++++++++- src/utils/entityIdMapper.ts | 21 ++ .../entity-id-mapper-u32-ceiling.test.ts | 28 ++ 4 files changed, 391 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index b563d2df..f1b33d21 100644 --- a/src/index.ts +++ b/src/index.ts @@ -180,6 +180,28 @@ export type { // Optional provider capability for generation-aware native indexes export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' +// Optional native graph-acceleration engine (cor 3.0) — the published provider +// contract + its columnar wire types. Brainy feature-detects an implementation +// and falls back to its pure-TS adjacency when absent. +export type { + GraphAccelerationProvider, + Subgraph, + OpaqueIdSet, + GraphTraversalDirection, + TraverseOptions, + EdgesForNodeOptions, + GraphCursorHandle, + GraphCursorOptions, + GraphCursorChunk, + GraphScores, + GraphComponents, + GraphPath, + PageRankOptions, + ComponentsOptions, + ShortestPathOptions, + NeighborhoodSampleOptions, + TopByDegreeOptions +} from './plugin.js' // Export embedding functionality import { diff --git a/src/plugin.ts b/src/plugin.ts index 100e5282..66ad7184 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -287,6 +287,298 @@ export interface GraphIndexProvider { getAllRelationshipCounts(): Map } +// ============= Graph Acceleration (optional native engine) ============= + +/** + * Opaque serialized id-set — a cor-internal roaring payload (a Node `Buffer` at + * runtime when the native engine is present), passed straight from a `find()` + * universe into `traverse` / `VectorIndexProvider.search` with NO id + * materialization in TypeScript (the O(1)-crossing query→expand win). Brainy + * NEVER inspects it; the provider version-tags the envelope and throws on a + * format mismatch (it owns integrity + the id-space-width guarantee). Typed as + * `Uint8Array` (which a Node `Buffer` satisfies) so the universal build stays + * browser-safe. + */ +export type OpaqueIdSet = Uint8Array + +/** Direction of a graph traversal relative to each frontier node. */ +export type GraphTraversalDirection = 'in' | 'out' | 'both' + +/** + * Columnar subgraph — the shared wire format for every native graph read + * (`traverse`, `edgesForNode`, cursor chunks). Parallel typed arrays, never an + * array-of-objects, so the NAPI boundary transfers them near-zero-copy and + * Brainy maps u64↔UUID **lazily** (only for the rows a caller actually renders, + * via `EntityIdMapperProvider.entityIntsToUuids` + `GraphIndexProvider.verbIntsToIds`). + * The three `edge*` arrays are parallel (index `i` is one edge); `nodes` / + * `nodeDepth` are parallel. + */ +export interface Subgraph { + /** Discovered entity ints; resolve via `entityIntsToUuids`. */ + nodes: BigInt64Array + /** Hop distance of each node from the nearest seed (parallel to `nodes`). Present iff `includeDepth`. */ + nodeDepth?: Uint8Array + /** Edge source entity ints. */ + edgeSources: BigInt64Array + /** Edge target entity ints. */ + edgeTargets: BigInt64Array + /** Edge verb ints; resolve via `verbIntsToIds`. */ + edgeVerbInts: BigInt64Array + /** Edge verb-type indices (stable TypeIdx; resolve via `TypeUtils.getVerbFromIndex`). */ + edgeTypes: Uint16Array + /** + * `true` when a `maxNodes` / `maxEdges` cap truncated the result. The returned + * rows are the deterministic BFS-order prefix; page the remainder with a + * `graphCursor` (traverse stays stateless/bounded — no continuation token). + */ + truncated?: boolean + /** When `truncated`, how many nodes/edges were cut (for "+N more" affordances). */ + truncatedNodeCount?: number + truncatedEdgeCount?: number +} + +/** Knobs shared by the bounded traversals. */ +export interface TraverseOptions { + /** Max hop distance from any seed. */ + depth: number + /** Edge direction to follow (default `'both'`). */ + direction?: GraphTraversalDirection + /** Restrict traversed edges to these verb-type indices (TypeIdx). */ + verbTypes?: number[] + /** Restrict traversed edges to these subtypes. */ + subtypes?: string[] + /** + * Visibility tiers to EXCLUDE from the frontier (default: `['internal','system']`, + * matching `related()`). Pass `[]` for an all-tiers (admin) view. + */ + excludeVisibility?: string[] + /** Cap on returned nodes (deterministic BFS-order prefix; sets `Subgraph.truncated`). */ + maxNodes?: number + /** Cap on returned edges. */ + maxEdges?: number + /** Include the edge arrays (`false` = nodes-only reachability). Default `true`. */ + includeEdges?: boolean + /** Populate `Subgraph.nodeDepth`. Default `false`. */ + includeDepth?: boolean +} + +/** Options for the both-direction single-node edge read. */ +export interface EdgesForNodeOptions { + /** Edge direction (default `'both'`). */ + direction?: GraphTraversalDirection + verbTypes?: number[] + subtypes?: string[] + excludeVisibility?: string[] + limit?: number +} + +/** Opaque server-side cursor handle (TTL-bounded; pinned to a generation at open). */ +export type GraphCursorHandle = string + +/** Options for opening a streaming whole-graph (or seeded) cursor. */ +export interface GraphCursorOptions { + direction?: GraphTraversalDirection + excludeVisibility?: string[] + /** `'light'` = ids/types only, no metadata-join columns (viz default); `'full'` = with joins. */ + projection?: 'light' | 'full' + /** Seed set to bound the walk; omitted = the whole graph. */ + seeds?: bigint[] | OpaqueIdSet + /** + * Resume token from a prior chunk — used to continue after a `SnapshotExpired` + * (the pinned generation's TTL lapsed): reopen with this to resume the walk. + */ + cursor?: string +} + +/** One streamed chunk of a graph cursor walk. */ +export interface GraphCursorChunk { + /** The chunk's nodes + edges, columnar. */ + subgraph: Subgraph + /** Opaque resume token (pass to `graphCursorOpen({ cursor })` after `SnapshotExpired`). */ + cursor?: string + /** `true` once the walk is exhausted; `graphCursorNext` should not be called again. */ + done: boolean +} + +/** Per-node scalar scores (pageRank, topByDegree); `scores[i]` belongs to `nodeInts[i]`. */ +export interface GraphScores { + nodeInts: BigInt64Array + scores: Float64Array +} + +/** Connected-component labelling; `componentIds[i]` is the component of `nodeInts[i]`. */ +export interface GraphComponents { + nodeInts: BigInt64Array + componentIds: Uint32Array + componentCount: number +} + +/** A shortest path: the node sequence + the edges between them (or `null` if unreachable). */ +export interface GraphPath { + nodeInts: BigInt64Array + edgeVerbInts: BigInt64Array + /** Sum of edge weights for a weighted path; hop count for an unweighted one. */ + cost: number +} + +export interface PageRankOptions { + /** Damping factor (default 0.85). */ + damping?: number + /** Max power-iterations (default ~20). */ + maxIterations?: number + /** Convergence tolerance. */ + tolerance?: number + excludeVisibility?: string[] + /** Return only the top-K by score (default: all). */ + topK?: number +} + +export interface ComponentsOptions { + /** `'weak'` ignores direction (default); `'strong'` honors it. */ + mode?: 'weak' | 'strong' + excludeVisibility?: string[] +} + +export interface ShortestPathOptions { + direction?: GraphTraversalDirection + /** `'edge'` = weighted by edge weight; `'hops'` = unweighted BFS (default `'hops'`). */ + weight?: 'edge' | 'hops' + verbTypes?: number[] + excludeVisibility?: string[] + /** Abandon the search past this many hops. */ + maxDepth?: number +} + +export interface NeighborhoodSampleOptions { + depth: number + direction?: GraphTraversalDirection + /** Max neighbors to sample per node (random, seeded for reproducibility). */ + fanout: number + /** RNG seed so a given (seeds, opts, seed) yields a stable sample for viz. */ + seed?: number + excludeVisibility?: string[] + maxNodes?: number +} + +export interface TopByDegreeOptions { + direction?: GraphTraversalDirection + topK: number + excludeVisibility?: string[] +} + +/** + * The `'graphAcceleration'` provider — an OPTIONAL native graph engine (the + * cor 3.0 acceleration layer). Brainy feature-detects it on the registered + * providers: when present, the public `brain.graph.*` surface and the + * `related({ node })` / `neighbors({ depth })` / `find({ connected })` paths + * route here; when absent, Brainy serves the same operations from its pure-TS + * adjacency fallback (correct, small-graph-scale). It is SEPARATE from the + * required {@link GraphIndexProvider} (which stays the bigint-adjacency + * contract) so "optional native acceleration" is explicit at the seam. + * + * **Generation-aware (8.0 time-travel).** Every read takes an optional trailing + * `generation?: bigint`; omitted = the current generation ("now"). With a + * generation, the provider resolves the graph **as of** that generation + * (`db.asOf(g).graph.*`), reading its generation-filtered adjacency rather than + * the now-only fast path. + * + * **Columnar + opaque-id-set.** Reads return the columnar {@link Subgraph}; + * `traverse` / sample accept seeds as `bigint[]` OR an {@link OpaqueIdSet} + * (a `find()` universe forwarded with no id materialization — the query→expand + * fusion). Visibility tiers are excluded at the index by default. + */ +export interface GraphAccelerationProvider { + /** `false` until the native engine has loaded; Brainy probes before routing. */ + readonly isInitialized: boolean + + /** + * @description Bounded multi-hop expansion from `seeds`, returning the reachable + * subgraph. One call replaces Brainy's per-hop BFS round-trips. + * @param seeds - Start nodes as entity ints, OR an {@link OpaqueIdSet} (a + * `find()` universe — the frontier is intersected in id-space, zero crossing). + * @param options - Depth, direction, edge/visibility filters, and caps. + * @param generation - Optional as-of generation (omitted = now). + * @returns The reachable {@link Subgraph} (BFS-order; `truncated` set if capped). + */ + traverse(seeds: bigint[] | OpaqueIdSet, options: TraverseOptions, generation?: bigint): Promise + + /** + * @description All edges incident to one node, both directions, as structure + + * cor-indexed fields (Brainy hydrates full edge metadata lazily — the provider + * cannot recover verb-id strings from its interned form). + * @param nodeInt - The node's interned entity int. + * @param options - Direction, edge/visibility filters, limit. + * @param generation - Optional as-of generation. + * @returns A {@link Subgraph} of the node's incident edges. + */ + edgesForNode(nodeInt: bigint, options: EdgesForNodeOptions, generation?: bigint): Promise + + /** + * @description Open a snapshot-consistent streaming walk of the whole graph (or a + * seeded region) for O(N) viz loads. Pins a generation at open; the walk reads + * as-of that pin (no dup/skip under concurrent writes). Handles are TTL-bounded — + * `graphCursorNext` after expiry rejects with `SnapshotExpired`; reopen with the + * last chunk's `cursor` to resume. Call `graphCursorClose` on normal completion. + * @param options - Direction, visibility, projection mode, optional seeds / resume cursor. + * @param generation - Optional explicit as-of generation to pin (else pins current). + * @returns A handle for `graphCursorNext` / `graphCursorClose`. + */ + graphCursorOpen(options: GraphCursorOptions, generation?: bigint): Promise + /** + * @description Pull the next chunk from an open cursor. + * @param handle - From `graphCursorOpen`. + * @param chunkSize - Target number of nodes/edges in this chunk. + * @returns The next {@link GraphCursorChunk}; `done: true` when exhausted. + */ + graphCursorNext(handle: GraphCursorHandle, chunkSize: number): Promise + /** + * @description Release an open cursor's pinned generation and server-side state. + * @param handle - From `graphCursorOpen`. + */ + graphCursorClose(handle: GraphCursorHandle): Promise + + /** + * @description PageRank over the visible graph (hidden tiers excluded by default, + * so importance reflects the public view). + * @param options - Damping, iterations, tolerance, visibility, optional top-K. + * @param generation - Optional as-of generation. + * @returns Per-node scores. + */ + pageRank(options: PageRankOptions, generation?: bigint): Promise + /** + * @description Connected components over the visible graph. + * @param options - Weak/strong, visibility. + * @param generation - Optional as-of generation. + * @returns Per-node component labels + component count. + */ + connectedComponents(options: ComponentsOptions, generation?: bigint): Promise + /** + * @description Shortest path between two nodes (weighted by edge weight or hop count). + * @param fromInt - Start node int. + * @param toInt - End node int. + * @param options - Direction, weighting, filters, max depth. + * @param generation - Optional as-of generation. + * @returns The path, or `null` if `toInt` is unreachable from `fromInt`. + */ + shortestPath(fromInt: bigint, toInt: bigint, options: ShortestPathOptions, generation?: bigint): Promise + /** + * @description Bounded, randomly-sampled neighborhood expansion (capped fan-out per + * node) — a representative subgraph for dense-graph viz. Seeded for reproducibility. + * @param seeds - Start nodes (ints or an {@link OpaqueIdSet}). + * @param options - Depth, fan-out, RNG seed, visibility, node cap. + * @param generation - Optional as-of generation. + * @returns The sampled {@link Subgraph}. + */ + neighborhoodSample(seeds: bigint[] | OpaqueIdSet, options: NeighborhoodSampleOptions, generation?: bigint): Promise + /** + * @description The top-K nodes by degree over the visible graph. + * @param options - Direction, K, visibility. + * @param generation - Optional as-of generation. + * @returns Per-node degree scores (descending). + */ + topByDegree(options: TopByDegreeOptions, generation?: bigint): Promise +} + /** * Optional capability interface for index providers that maintain versioned * (generation-aware) internal state — the provider-side half of Brainy 8.0's @@ -411,7 +703,20 @@ export interface VectorIndexProvider { queryVector: Vector, k?: number, filter?: (id: string) => Promise, - options?: { rerank?: { multiplier: number }; candidateIds?: string[] } + options?: { + rerank?: { multiplier: number } + candidateIds?: string[] + /** + * Predicate-pushdown universe — restrict the result to this id-set, applied + * INSIDE the beam walk (walk traverses all nodes, collects only allowed), which + * recovers the filtered recall that post-filtering loses. A native provider with + * cor as the metadata index gets the `find()` universe as an {@link OpaqueIdSet} + * (zero id materialization); the pure-JS path gets a `ReadonlySet`. Absent + * = no restriction. Optional — a provider that doesn't pushdown ignores it and + * falls back to `filter`. + */ + allowedIds?: OpaqueIdSet | ReadonlySet + } ): Promise> size(): number clear(): void @@ -437,6 +742,20 @@ export interface EntityIdMapperProvider { clear(): Promise getAllIntIds(): number[] intsIterableToUuids(ints: Iterable): string[] + /** + * @description Batch reverse-resolve u64 entity ints → UUID strings — the + * bigint counterpart of {@link EntityIdMapperProvider.intsIterableToUuids}, + * for the native graph engine ({@link GraphAccelerationProvider}, whose + * `Subgraph.nodes` is a `BigInt64Array`). Brainy resolves lazily — only the + * rows a caller renders — but viz/export render many at once, so one batch + * crossing beats N. Order-preserving (one entry per input). A native mapper + * resolves every int from its binary int↔uuid store; the JS fallback resolves + * assigned ints and yields `''` for a never-assigned int (which does not occur + * for graph-engine results, since those only reference assigned ints). + * @param nodeInts - Entity ints as returned in a {@link Subgraph}. + * @returns One UUID per input, in order. + */ + entityIntsToUuids(nodeInts: BigInt64Array): string[] readonly size: number } diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index 7a54846f..6551e0ed 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -291,6 +291,27 @@ export class EntityIdMapper implements EntityIdMapperProvider { return result } + /** + * @description Batch reverse-resolve u64 entity ints (a `BigInt64Array`) → UUID + * strings — the bigint counterpart of {@link EntityIdMapper.intsIterableToUuids}, + * for the native graph engine whose `Subgraph.nodes` is a `BigInt64Array`. The JS + * mapper is keyed by `number` (u32-era); each int is narrowed via `Number()` — safe + * for the JS fallback's id range. Order-preserving (one entry per input): an int the + * mapper has not assigned yields `''` (does not occur for graph-engine results, which + * reference assigned ints only). + * @param nodeInts - Entity ints as returned in a graph `Subgraph`. + * @returns One UUID per input, in order (`''` for a never-assigned int). + * @example + * const uuids = mapper.entityIntsToUuids(subgraph.nodes) + */ + entityIntsToUuids(nodeInts: BigInt64Array): string[] { + const result: string[] = new Array(nodeInts.length) + for (let i = 0; i < nodeInts.length; i++) { + result[i] = this.intToUuid.get(Number(nodeInts[i])) ?? '' + } + return result + } + /** * Flush mappings to storage */ diff --git a/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts b/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts index 0b92f768..2f9e8f9b 100644 --- a/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts +++ b/tests/unit/utils/entity-id-mapper-u32-ceiling.test.ts @@ -96,3 +96,31 @@ describe('EntityIdMapper U32 IdSpace ceiling (Brainy 8.0)', () => { expect(m.getOrAssign('uuid-existing')).toBe(assigned) }) }) + +describe('EntityIdMapper.entityIntsToUuids (bigint batch resolver — graph engine)', () => { + it('reverse-resolves a BigInt64Array of assigned ints to UUIDs, in order', async () => { + const m = makeMapper() + await m.init() + const a = m.getOrAssign('uuid-a') + const b = m.getOrAssign('uuid-b') + const c = m.getOrAssign('uuid-c') + // Out-of-assignment order, to prove it's positional (not sorted). + const ints = BigInt64Array.from([BigInt(c), BigInt(a), BigInt(b)]) + expect(m.entityIntsToUuids(ints)).toEqual(['uuid-c', 'uuid-a', 'uuid-b']) + }) + + it('yields "" for a never-assigned int (order-preserving, no drop)', async () => { + const m = makeMapper() + await m.init() + const a = m.getOrAssign('uuid-a') + const ints = BigInt64Array.from([BigInt(a), 999999n]) + // Position is preserved — the unknown int does not collapse the array. + expect(m.entityIntsToUuids(ints)).toEqual(['uuid-a', '']) + }) + + it('returns an empty array for empty input', async () => { + const m = makeMapper() + await m.init() + expect(m.entityIntsToUuids(new BigInt64Array(0))).toEqual([]) + }) +})