soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): brain.graph.export() + noun-walk cursor + noun visibility hydration

Browse source 
brain.graph.export() streams the WHOLE graph in one O(N+E) pass — node chunks
then edge chunks, async-iterable — the right primitive for visualizing all data
(vs. paging per node). Native snapshot-consistent graphCursor when present, else
a TS cursor walk over nouns + verbs. Building it surfaced two latent noun bugs,
both fixed here (each a correctness win beyond export):

- getNounsWithPagination IGNORED its cursor ('offset-based, cursor planned') —
  the noun mirror of the verb bug fixed in the cursor-pagination commit. Latent
  because the only multi-page consumer (aggregate backfill) uses one big page; a
  small chunkSize needed page 2 and, trusting the returned-but-ignored nextCursor,
  re-fetched page 0 forever (infinite loop). Ported the proven verb cursor:
  opaque cn1:<shard>:<id> token, stable within-shard id order, resume-after,
  O(N) at any chunk size. (Generalized verbIdFromVectorPath → idFromVectorPath.)
- hydrateNounWithMetadata DROPPED 'visibility' (noun mirror of the Fix #1 verb
  hydration bug) — so getNouns-fed visibility filters saw nothing and leaked
  system (VFS root) / internal nodes. Now hydrated.

- New: GraphApi.export + GraphExportOptions (chunkSize, includeInternal/System,
  includeNodes/Edges). hydrateNativeSubgraph extracted + shared by subgraph+export.

Test: graph-export.test.ts — full-graph completeness incl. isolated nodes,
default-hides-internal, node/edge include toggles, chunkSize chunking (the
case that exposed the cursor hang). Full gate green.
This commit is contained in:
David Snelling 2026-06-21 10:22:12 -07:00
parent 8c2b57a488
commit c2a84c9d1f
4 changed files with 369 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

134
src/brainy.ts
View file

@ -37,7 +37,7 @@ import { setGlobalCache } from './utils/unifiedCache.js'
import type { UnifiedCache } from './utils/unifiedCache.js'
import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js'
import { PluginRegistry, isVersionedIndexProvider, isGraphAccelerationProvider } from './plugin.js'
import type { GraphAccelerationProvider, TraverseOptions } from './plugin.js'
import type { GraphAccelerationProvider, TraverseOptions, Subgraph } from './plugin.js'
import type {
BrainyPlugin,
BrainyPluginContext,
@ -93,6 +93,7 @@ import {
GraphView,
GraphNode,
SubgraphOptions,
GraphExportOptions,
GetOptions,
AddManyParams,
RemoveManyParams,
@ -3178,7 +3179,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
get graph(): GraphApi<T> {
return {
subgraph: (seeds: string | string[], options?: SubgraphOptions) =>
this.graphSubgraph(seeds, options)
this.graphSubgraph(seeds, options),
export: (options?: GraphExportOptions) => this.graphExport(options)
}
}
@ -3255,7 +3257,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
frontier = next
}
return this.buildGraphView(nodeDepth, [...edgesById.values()], options, truncated)
return this.buildGraphView(
nodeDepth,
[...edgesById.values()],
options?.hydrateNodes !== false,
truncated
)
}
/** One frontier hop's incident edges in the requested direction (filters applied by `related()`). */
@ -3318,16 +3325,26 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
const sub = await accel.traverse(seedInts, traverseOptions)
return this.hydrateNativeSubgraph(sub, options?.hydrateNodes !== false)
}
// Edge hydration: verb ints → ids → Relation objects.
/**
* @description Hydrate a columnar native `Subgraph` into a {@link GraphView}:
* edge verb-ints → `Relation`s (via `verbIntsToIds` + `getVerbsBatchCached`),
* and node ints paired with their depth (position-preserving — `entityIntsToUuids`
* drops deleted ints, which would desync the depth column). Shared by the native
* `subgraph` and `export` paths.
* @param sub - The provider's columnar subgraph / cursor chunk.
* @param hydrateNodes - Whether to batch-load each node's `type` / `subtype`.
* @returns The hydrated {@link GraphView}.
*/
private async hydrateNativeSubgraph(sub: Subgraph, hydrateNodes: boolean): Promise<GraphView<T>> {
const verbIds = (await this.graphIndex.verbIntsToIds(Array.from(sub.edgeVerbInts))).filter(
(id): id is string => id !== null
)
const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds)
const edges = this.verbsToRelations([...verbsMap.values()])
// Node hydration: pair each node int with its depth (position-preserving —
// entityIntsToUuids drops deleted ints, which would desync the depth column).
const idMapper = this.metadataIndex.getIdMapper()
const nodeDepth = new Map<string, number>()
for (let i = 0; i < sub.nodes.length; i++) {
@ -3335,7 +3352,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (uuid !== undefined) nodeDepth.set(uuid, sub.nodeDepth ? sub.nodeDepth[i] : 0)
}
return this.buildGraphView(nodeDepth, edges, options, sub.truncated ?? false)
return this.buildGraphView(nodeDepth, edges, hydrateNodes, sub.truncated ?? false)
}
/**
@ -3346,12 +3363,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
private async buildGraphView(
nodeDepth: Map<string, number>,
edges: Relation<T>[],
options: SubgraphOptions | undefined,
hydrateNodes: boolean,
truncated: boolean
): Promise<GraphView<T>> {
const ids = [...nodeDepth.keys()]
let entities: Map<string, Entity<T>> | undefined
if (options?.hydrateNodes !== false && ids.length > 0) {
if (hydrateNodes && ids.length > 0) {
entities = await this.batchGet(ids)
}
const nodes: GraphNode[] = ids.map((id) => {
@ -3368,6 +3385,105 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return { nodes, edges, truncated }
}
/**
* @description {@link GraphApi.export} dispatch: native snapshot-consistent
* graph cursor when present, else the TS cursor walk over nouns + verbs. Both
* are O(N+E) single passes (cursor pagination — no re-scan).
*/
private async *graphExport(options?: GraphExportOptions): AsyncGenerator<GraphView<T>> {
await this.ensureInitialized()
const accel = this.graphAccelerationProvider()
if (accel) {
yield* this.graphExportNative(accel, options)
} else {
yield* this.graphExportFallback(options)
}
}
/**
* @description TS export: stream all nouns (node chunks) then all verbs (edge
* chunks) via cursor pagination — O(N+E), no re-scan. Node refs carry
* `type`/`subtype` straight from the noun records (no extra read). Hidden tiers
* are excluded by default.
*/
private async *graphExportFallback(options?: GraphExportOptions): AsyncGenerator<GraphView<T>> {
const chunkSize = options?.chunkSize ?? 1000
const excludedTiers = this.excludedVisibilityTiers(options ?? {})
const excludedSet = excludedTiers ? new Set<string>(excludedTiers) : null
if (options?.includeNodes !== false) {
let cursor: string | undefined
let offset = 0
for (;;) {
const page = await this.storage.getNouns({
pagination: cursor ? { limit: chunkSize, cursor } : { limit: chunkSize, offset }
})
const nodes: GraphNode[] = []
for (const noun of page.items) {
if (excludedSet && noun.visibility && excludedSet.has(noun.visibility)) continue
nodes.push({
id: noun.id,
type: noun.type,
...(noun.subtype !== undefined && { subtype: noun.subtype })
})
}
if (nodes.length > 0) yield { nodes, edges: [], truncated: false }
if (!page.hasMore || page.items.length === 0) break
if (page.nextCursor) cursor = page.nextCursor
else offset += page.items.length
}
}
if (options?.includeEdges !== false) {
const filter = excludedTiers
? { excludeVisibility: excludedTiers as unknown as string[] }
: undefined
let cursor: string | undefined
let offset = 0
for (;;) {
const page = await this.storage.getVerbs({
pagination: cursor ? { limit: chunkSize, cursor } : { limit: chunkSize, offset },
filter
})
if (page.items.length > 0) {
yield { nodes: [], edges: this.verbsToRelations(page.items), truncated: false }
}
if (!page.hasMore || page.items.length === 0) break
if (page.nextCursor) cursor = page.nextCursor
else offset += page.items.length
}
}
}
/**
* @description Native export: open a snapshot-consistent graph cursor (pins a
* generation — no dup/skip under concurrent writes), pull light chunks, and
* hydrate each into a {@link GraphView}. The cursor is always closed (even on
* early break / error). Cross-layer tested against the native provider; the TS
* fallback exercises the same chunk shape in CI.
*/
private async *graphExportNative(
accel: GraphAccelerationProvider,
options?: GraphExportOptions
): AsyncGenerator<GraphView<T>> {
const excludedTiers = this.excludedVisibilityTiers(options ?? {})
const handle = await accel.graphCursorOpen({
direction: 'both',
...(excludedTiers && { excludeVisibility: excludedTiers as unknown as string[] }),
projection: 'light'
})
try {
for (;;) {
const chunk = await accel.graphCursorNext(handle, options?.chunkSize ?? 1000)
const view = await this.hydrateNativeSubgraph(chunk.subgraph, true)
if (view.nodes.length > 0 || view.edges.length > 0) yield view
if (chunk.done) break
}
} finally {
await accel.graphCursorClose(handle)
}
}
// ============= SEARCH & DISCOVERY =============
/**