soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: includeHidden — export carries every visibility tier for migration-grade canon completeness

Browse source
This commit is contained in:
David Snelling 2026-07-27 11:22:25 -07:00
parent 3e4a17dcdf
commit 63c1eeb902
4 changed files with 205 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/db/db.ts
View file

@ -528,6 +528,11 @@ export class Db<T = any> {
* throws {@link CanonicalEnumerationUnavailableError} rather than silently mixing
* generations or missing the overlay's own entities.
*
* `options.includeHidden: true` admits BOTH hidden visibility tiers
* (`'internal'` and `'system'`) into a whole-brain/predicate export, in EITHER
* `enumeration` mode — see {@link ExportOptions.includeHidden}. Migration-grade
* exports set this; consumer-facing exports leave it off (default: false).
*
* @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}.
* @param options - HOW to export (vectors / VFS bytes / edge policy / enumeration mode). See {@link ExportOptions}.
* @returns A versioned, portable `PortableGraph` document.

123
src/db/portableGraph.ts
View file

@ -94,6 +94,22 @@ export interface ExportOptions {
includeContent?: boolean
/** Include `visibility:'system'` entities (e.g. the VFS root) (default: false). */
includeSystem?: boolean
/**
* Admit BOTH hidden visibility tiers — `'internal'` AND `'system'` — into the
* whole-brain/predicate candidate set, in EITHER `enumeration` mode (default:
* false — today's behavior is byte-identical). `includeSystem` alone only ever
* reached `'system'` for a structural selector's per-entity gate; whole-brain/
* predicate enumeration never forwarded it into the candidate walk at all, so a
* hidden-tier row could never survive a whole-brain export regardless of any
* flag — the gap this option closes. `includeHidden: true` IMPLIES
* `includeSystem: true` (both tiers are admitted together; there is no
* "system but not internal" combination via this flag) — `includeSystem` on
* its own keeps its narrower, pre-existing meaning for back-compat.
*
* Migration-grade exports set `includeHidden: true` — a complete-canon export
* must carry every visibility tier; consumer-facing exports leave it off.
*/
includeHidden?: boolean
/** Which edges to include (default: `'induced'`). */
edges?: 'induced' | 'incident' | 'none'
/**
@ -358,6 +374,7 @@ export async function exportGraph<T = any>(
includeVectors = false,
includeContent = false,
includeSystem = false,
includeHidden = false,
edges = 'induced',
enumeration = 'index',
reportIndexDrift = false
@ -371,15 +388,23 @@ export async function exportGraph<T = any>(
)
}
const wantDrift = enumeration === 'canonical' && reportIndexDrift
// includeHidden IMPLIES includeSystem (see ExportOptions.includeHidden's JSDoc) — every
// system-tier gate below reads THIS combined value, never the raw option, so
// `includeHidden` alone is always sufficient to see system-tier rows too.
const effectiveIncludeSystem = includeSystem || includeHidden
// 1. Resolve the node-id set (+ the index's raw candidate set, only when diffing it).
// Both `enumerateAllCanonical` and `enumerateAllIndexed` receive the SAME
// `effectiveIncludeSystem`/`includeHidden` pair below, so a drift diff can never
// contain tier-policy noise — only genuine index-vs-canonical disagreement.
const { idSet, indexCandidateIds } = await resolveSelector(
reader,
storage,
selector,
includeSystem,
effectiveIncludeSystem,
enumeration,
wantDrift
wantDrift,
includeHidden
)
// 2. Read canonical entities (reserved fields top-level), applying any predicate filter.
@ -392,7 +417,7 @@ export async function exportGraph<T = any>(
for (const id of idSet) {
const e = await reader.get(id, { includeVectors })
if (!e) continue
if (!includeSystem && (e as any).visibility === 'system') continue
if (!effectiveIncludeSystem && (e as any).visibility === 'system') continue
if (usePredicate && !matchesPredicate(e, selector)) continue
entityMap.set(id, e)
entities.push(toPortableGraphEntity(e, includeVectors))
@ -422,8 +447,8 @@ export async function exportGraph<T = any>(
// relation regardless of how the node set was produced.
const { relations, danglingIds } =
enumeration === 'canonical'
? await collectEdgesCanonical(storage!, keptIds, edges)
: await collectEdges(reader, keptIds, edges)
? await collectEdgesCanonical(storage!, keptIds, edges, effectiveIncludeSystem, includeHidden)
: await collectEdges(reader, keptIds, edges, includeHidden)
// 4. VFS blob bytes (only when requested).
let blobs: Record<string, string> | undefined
@ -598,7 +623,9 @@ function hasPredicate(s: ExportSelector): boolean {
* @param storage - Storage adapter (only touched when `enumeration:'canonical'`
* resolves the whole-brain/predicate branch).
* @param s - The export selector.
* @param includeSystem - Whether `visibility:'system'` entities are wanted.
* @param includeSystem - The ALREADY-COMBINED `includeSystem || includeHidden` value
* (see `exportGraph`'s `effectiveIncludeSystem`) — whether `visibility:'system'`
* entities are wanted.
* @param enumeration - `'index'` (default) or `'canonical'` — see {@link ExportOptions.enumeration}.
* Only affects the whole-brain/predicate branch (the `else` below): structural
* selectors (`ids`/`collection`/`connected`/`vfsPath`) never rode the metadata
@ -606,6 +633,9 @@ function hasPredicate(s: ExportSelector): boolean {
* @param wantIndexCandidates - When true (only meaningful with `enumeration:'canonical'`
* on the whole-brain/predicate branch), ALSO run the index-based walk and return
* its raw candidate set as `indexCandidateIds`, for {@link ExportIndexDrift}.
* @param includeHidden - Whether `visibility:'internal'` entities are ALSO wanted
* (see {@link ExportOptions.includeHidden}). Threaded to BOTH enumeration
* functions identically so a drift diff never contains tier-policy noise.
*/
async function resolveSelector<T>(
reader: PortableGraphReader<T>,
@ -613,7 +643,8 @@ async function resolveSelector<T>(
s: ExportSelector,
includeSystem: boolean,
enumeration: 'index' | 'canonical',
wantIndexCandidates: boolean
wantIndexCandidates: boolean,
includeHidden: boolean
): Promise<{ idSet: Set<string>; indexCandidateIds?: Set<string> }> {
let idSet: Set<string>
let indexCandidateIds: Set<string> | undefined
@ -626,10 +657,10 @@ async function resolveSelector<T>(
} else if (s.vfsPath) {
idSet = await resolveVfsPath(reader, s.vfsPath, s.recursive ?? true, s.depth)
} else if (enumeration === 'canonical') {
idSet = await enumerateAllCanonical(storage!)
if (wantIndexCandidates) indexCandidateIds = await enumerateAllIndexed(reader, s)
idSet = await enumerateAllCanonical(storage!, includeSystem, includeHidden)
if (wantIndexCandidates) indexCandidateIds = await enumerateAllIndexed(reader, s, includeHidden)
} else {
idSet = await enumerateAllIndexed(reader, s)
idSet = await enumerateAllIndexed(reader, s, includeHidden)
}
if (!includeSystem) idSet.delete(VFS_ROOT_ID)
return { idSet, indexCandidateIds }
@ -640,13 +671,30 @@ async function resolveSelector<T>(
* paginated `find()`. The metadata index is an acceleration structure over
* this candidate set — see {@link enumerateAllCanonical} for the storage-level
* counterpart that never consults it.
*
* @param includeHidden - When true, forwards `includeInternal: true` AND
* `includeSystem: true` into the SAME `find()` call — `find()` supports both
* flags simultaneously (confirmed via `FindParams.includeInternal`/`includeSystem`
* and `Brainy`'s `resolveHiddenIds`/`excludedVisibilityTiers`), so ONE pass
* reaches both hidden tiers; no per-tier union pass is needed. When false
* (default), neither flag is forwarded — the pre-existing behavior, preserved
* byte-identically for back-compat (`ExportOptions.includeSystem` alone never
* reached this far; see {@link ExportOptions.includeHidden}'s JSDoc).
*/
async function enumerateAllIndexed<T>(reader: PortableGraphReader<T>, s: ExportSelector): Promise<Set<string>> {
async function enumerateAllIndexed<T>(
reader: PortableGraphReader<T>,
s: ExportSelector,
includeHidden = false
): Promise<Set<string>> {
const params: any = {}
if (s.type !== undefined) params.type = s.type
if (s.subtype !== undefined) params.subtype = s.subtype
if (s.where !== undefined) params.where = s.where
if (s.service !== undefined) params.service = s.service
if (includeHidden) {
params.includeInternal = true
params.includeSystem = true
}
const ids = new Set<string>()
let offset = 0
// eslint-disable-next-line no-constant-condition
@ -672,13 +720,18 @@ async function enumerateAllIndexed<T>(reader: PortableGraphReader<T>, s: ExportS
* path does, so both paths share one predicate-evaluation code path and can only
* disagree on candidacy, never on what a match means.
*
* Mirrors `find()`'s default hidden-tier policy (always hides `'internal'` and
* `'system'` here — `enumerateAllIndexed` never opts either back in via `find()`
* either, since `ExportOptions.includeSystem` is applied later, per-entity, and
* only reachable for ids a selector already named directly) so the two
* enumeration modes produce identical id sets when the index is healthy.
* Mirrors `find()`'s hidden-tier policy given the SAME `includeSystem`/`includeHidden`
* pair (see {@link enumerateAllIndexed}) so the two enumeration modes produce
* identical id sets when the index is healthy, at ANY tier-visibility setting.
*
* @param includeSystem - The ALREADY-COMBINED `includeSystem || includeHidden` value.
* @param includeHidden - Whether `'internal'`-tier nouns are ALSO admitted.
*/
async function enumerateAllCanonical(storage: StorageAdapter): Promise<Set<string>> {
async function enumerateAllCanonical(
storage: StorageAdapter,
includeSystem = false,
includeHidden = false
): Promise<Set<string>> {
const ids = new Set<string>()
let offset = 0
let cursor: string | undefined
@ -686,7 +739,8 @@ async function enumerateAllCanonical(storage: StorageAdapter): Promise<Set<strin
while (true) {
const page = await storage.getNouns({ pagination: { limit: ENUM_PAGE, offset, cursor } })
for (const item of page.items) {
if (item.visibility === 'internal' || item.visibility === 'system') continue
if (item.visibility === 'internal' && !includeHidden) continue
if (item.visibility === 'system' && !includeSystem) continue
ids.add(item.id)
}
if (!page.hasMore || page.items.length === 0) break
@ -860,19 +914,27 @@ function toPortableGraphRelation<T>(r: Relation<T>): PortableGraphRelation {
return br
}
/**
* @param includeHidden - When true, forwards `includeInternal`/`includeSystem` into
* every `related()` call so hidden-tier relations reach candidacy too — mirrors
* {@link enumerateAllIndexed}'s `includeHidden` handling, and preserves back-compat
* when false/omitted (the pre-existing, unconditional hidden-tier exclusion).
*/
async function collectEdges<T>(
reader: PortableGraphReader<T>,
idSet: Set<string>,
edges: 'induced' | 'incident' | 'none'
edges: 'induced' | 'incident' | 'none',
includeHidden = false
): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> {
if (edges === 'none') return { relations: [] }
const tierOptIn = includeHidden ? { includeInternal: true, includeSystem: true } : {}
const relations: PortableGraphRelation[] = []
const dangling = new Set<string>()
const seen = new Set<string>()
for (const id of idSet) {
const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT })
const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT, ...tierOptIn })
for (const r of rels) {
if (seen.has(r.id)) continue
const toIn = idSet.has(r.to)
@ -885,7 +947,7 @@ async function collectEdges<T>(
if (edges === 'incident') {
for (const id of idSet) {
const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT })
const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT, ...tierOptIn })
for (const r of rels) {
if (seen.has(r.id)) continue
if (!idSet.has(r.from)) {
@ -921,15 +983,21 @@ function hnswVerbToPortableGraphRelation(v: HNSWVerbWithMetadata): PortableGraph
* branch: relations can be blinded by adjacency-index corruption regardless of
* how `idSet` (the kept node ids) was produced.
*
* Mirrors `related()`'s default hidden-tier policy (always hides `'internal'`
* and `'system'` — `collectEdges` never opts either back in via `related()`
* either) so the two enumeration modes produce identical relation sets when the
* index is healthy.
* Mirrors `related()`'s default hidden-tier policy (hides `'internal'` and
* `'system'` unless `includeHidden`/`includeSystem` say otherwise) so the two
* enumeration modes produce identical relation sets when the index is healthy.
*
* @param includeSystem - Whether `'system'`-tier verbs are admitted (the
* caller passes the ALREADY-combined `includeSystem || includeHidden` value —
* see `exportGraph`'s `effectiveIncludeSystem`).
* @param includeHidden - Whether `'internal'`-tier verbs are ALSO admitted.
*/
async function collectEdgesCanonical(
storage: StorageAdapter,
idSet: Set<string>,
edges: 'induced' | 'incident' | 'none'
edges: 'induced' | 'incident' | 'none',
includeSystem = false,
includeHidden = false
): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> {
if (edges === 'none') return { relations: [] }
@ -944,7 +1012,8 @@ async function collectEdgesCanonical(
const page = await storage.getVerbs({ pagination: { limit: ENUM_PAGE, offset, cursor } })
for (const v of page.items) {
if (seen.has(v.id)) continue
if (v.visibility === 'internal' || v.visibility === 'system') continue
if (v.visibility === 'internal' && !includeHidden) continue
if (v.visibility === 'system' && !includeSystem) continue
const fromIn = idSet.has(v.sourceId)
const toIn = idSet.has(v.targetId)
if (edges === 'induced') {