feat: includeHidden — export carries every visibility tier for migration-grade canon completeness

RELEASES.md

@@ -61,6 +61,15 @@ to the caller today.
   Migration-audit evidence, not a repair: nonzero drift is reported loudly
   (`console.warn` with the counts) and nothing is auto-healed — run `brain.repairIndex()`
   to reconcile the metadata index once drift is confirmed.
+- **New: `export(selector, { includeHidden: true })`** (default: false — unchanged
+  behavior). Without it, a whole-brain/predicate export could never carry a
+  `visibility:'internal'` or `'system'` row, in EITHER `enumeration` mode — a real gap
+  for a bulk-migration fold auditing per-visibility-tier, where a hidden tier is real
+  user data, not noise to drop. `includeHidden` admits both tiers into candidacy in
+  both modes (and implies `includeSystem`; `includeSystem` alone keeps its narrower,
+  pre-existing meaning). **Migration-grade exports set `includeHidden: true`** — a
+  complete-canon export must carry every visibility tier; consumer-facing exports
+  leave it off.
 - **Ops note (consumer-invisible): the release pipeline's forge-registry publish now runs
   on CI**, triggered by the release tag, instead of PUTting the tarball from the laptop
   over WAN — no change to what gets published or how a consumer installs it.

src/db/db.ts

@@ -528,6 +528,11 @@ export class Db<T = any> {
    * throws {@link CanonicalEnumerationUnavailableError} rather than silently mixing
    * generations or missing the overlay's own entities.
    *
+   * `options.includeHidden: true` admits BOTH hidden visibility tiers
+   * (`'internal'` and `'system'`) into a whole-brain/predicate export, in EITHER
+   * `enumeration` mode — see {@link ExportOptions.includeHidden}. Migration-grade
+   * exports set this; consumer-facing exports leave it off (default: false).
+   *
    * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}.
    * @param options - HOW to export (vectors / VFS bytes / edge policy / enumeration mode). See {@link ExportOptions}.
    * @returns A versioned, portable `PortableGraph` document.

src/db/portableGraph.ts

@@ -94,6 +94,22 @@ export interface ExportOptions {
   includeContent?: boolean
   /** Include `visibility:'system'` entities (e.g. the VFS root) (default: false). */
   includeSystem?: boolean
+  /**
+   * Admit BOTH hidden visibility tiers — `'internal'` AND `'system'` — into the
+   * whole-brain/predicate candidate set, in EITHER `enumeration` mode (default:
+   * false — today's behavior is byte-identical). `includeSystem` alone only ever
+   * reached `'system'` for a structural selector's per-entity gate; whole-brain/
+   * predicate enumeration never forwarded it into the candidate walk at all, so a
+   * hidden-tier row could never survive a whole-brain export regardless of any
+   * flag — the gap this option closes. `includeHidden: true` IMPLIES
+   * `includeSystem: true` (both tiers are admitted together; there is no
+   * "system but not internal" combination via this flag) — `includeSystem` on
+   * its own keeps its narrower, pre-existing meaning for back-compat.
+   *
+   * Migration-grade exports set `includeHidden: true` — a complete-canon export
+   * must carry every visibility tier; consumer-facing exports leave it off.
+   */
+  includeHidden?: boolean
   /** Which edges to include (default: `'induced'`). */
   edges?: 'induced' | 'incident' | 'none'
   /**
@@ -358,6 +374,7 @@ export async function exportGraph<T = any>(
     includeVectors = false,
     includeContent = false,
     includeSystem = false,
+    includeHidden = false,
     edges = 'induced',
     enumeration = 'index',
     reportIndexDrift = false
@@ -371,15 +388,23 @@ export async function exportGraph<T = any>(
     )
   }
   const wantDrift = enumeration === 'canonical' && reportIndexDrift
+  // includeHidden IMPLIES includeSystem (see ExportOptions.includeHidden's JSDoc) — every
+  // system-tier gate below reads THIS combined value, never the raw option, so
+  // `includeHidden` alone is always sufficient to see system-tier rows too.
+  const effectiveIncludeSystem = includeSystem || includeHidden
 
   // 1. Resolve the node-id set (+ the index's raw candidate set, only when diffing it).
+  // Both `enumerateAllCanonical` and `enumerateAllIndexed` receive the SAME
+  // `effectiveIncludeSystem`/`includeHidden` pair below, so a drift diff can never
+  // contain tier-policy noise — only genuine index-vs-canonical disagreement.
   const { idSet, indexCandidateIds } = await resolveSelector(
     reader,
     storage,
     selector,
-    includeSystem,
+    effectiveIncludeSystem,
     enumeration,
-    wantDrift
+    wantDrift,
+    includeHidden
   )
 
   // 2. Read canonical entities (reserved fields top-level), applying any predicate filter.
@@ -392,7 +417,7 @@ export async function exportGraph<T = any>(
   for (const id of idSet) {
     const e = await reader.get(id, { includeVectors })
     if (!e) continue
-    if (!includeSystem && (e as any).visibility === 'system') continue
+    if (!effectiveIncludeSystem && (e as any).visibility === 'system') continue
     if (usePredicate && !matchesPredicate(e, selector)) continue
     entityMap.set(id, e)
     entities.push(toPortableGraphEntity(e, includeVectors))
@@ -422,8 +447,8 @@ export async function exportGraph<T = any>(
   // relation regardless of how the node set was produced.
   const { relations, danglingIds } =
     enumeration === 'canonical'
-      ? await collectEdgesCanonical(storage!, keptIds, edges)
+      ? await collectEdgesCanonical(storage!, keptIds, edges, effectiveIncludeSystem, includeHidden)
-      : await collectEdges(reader, keptIds, edges)
+      : await collectEdges(reader, keptIds, edges, includeHidden)
 
   // 4. VFS blob bytes (only when requested).
   let blobs: Record<string, string> | undefined
@@ -598,7 +623,9 @@ function hasPredicate(s: ExportSelector): boolean {
  * @param storage - Storage adapter (only touched when `enumeration:'canonical'`
  *   resolves the whole-brain/predicate branch).
  * @param s - The export selector.
- * @param includeSystem - Whether `visibility:'system'` entities are wanted.
+ * @param includeSystem - The ALREADY-COMBINED `includeSystem || includeHidden` value
+ *   (see `exportGraph`'s `effectiveIncludeSystem`) — whether `visibility:'system'`
+ *   entities are wanted.
  * @param enumeration - `'index'` (default) or `'canonical'` — see {@link ExportOptions.enumeration}.
  *   Only affects the whole-brain/predicate branch (the `else` below): structural
  *   selectors (`ids`/`collection`/`connected`/`vfsPath`) never rode the metadata
@@ -606,6 +633,9 @@ function hasPredicate(s: ExportSelector): boolean {
  * @param wantIndexCandidates - When true (only meaningful with `enumeration:'canonical'`
  *   on the whole-brain/predicate branch), ALSO run the index-based walk and return
  *   its raw candidate set as `indexCandidateIds`, for {@link ExportIndexDrift}.
+ * @param includeHidden - Whether `visibility:'internal'` entities are ALSO wanted
+ *   (see {@link ExportOptions.includeHidden}). Threaded to BOTH enumeration
+ *   functions identically so a drift diff never contains tier-policy noise.
  */
 async function resolveSelector<T>(
   reader: PortableGraphReader<T>,
@@ -613,7 +643,8 @@ async function resolveSelector<T>(
   s: ExportSelector,
   includeSystem: boolean,
   enumeration: 'index' | 'canonical',
-  wantIndexCandidates: boolean
+  wantIndexCandidates: boolean,
+  includeHidden: boolean
 ): Promise<{ idSet: Set<string>; indexCandidateIds?: Set<string> }> {
   let idSet: Set<string>
   let indexCandidateIds: Set<string> | undefined
@@ -626,10 +657,10 @@ async function resolveSelector<T>(
   } else if (s.vfsPath) {
     idSet = await resolveVfsPath(reader, s.vfsPath, s.recursive ?? true, s.depth)
   } else if (enumeration === 'canonical') {
-    idSet = await enumerateAllCanonical(storage!)
+    idSet = await enumerateAllCanonical(storage!, includeSystem, includeHidden)
-    if (wantIndexCandidates) indexCandidateIds = await enumerateAllIndexed(reader, s)
+    if (wantIndexCandidates) indexCandidateIds = await enumerateAllIndexed(reader, s, includeHidden)
   } else {
-    idSet = await enumerateAllIndexed(reader, s)
+    idSet = await enumerateAllIndexed(reader, s, includeHidden)
   }
   if (!includeSystem) idSet.delete(VFS_ROOT_ID)
   return { idSet, indexCandidateIds }
@@ -640,13 +671,30 @@ async function resolveSelector<T>(
  * paginated `find()`. The metadata index is an acceleration structure over
  * this candidate set — see {@link enumerateAllCanonical} for the storage-level
  * counterpart that never consults it.
+ *
+ * @param includeHidden - When true, forwards `includeInternal: true` AND
+ *   `includeSystem: true` into the SAME `find()` call — `find()` supports both
+ *   flags simultaneously (confirmed via `FindParams.includeInternal`/`includeSystem`
+ *   and `Brainy`'s `resolveHiddenIds`/`excludedVisibilityTiers`), so ONE pass
+ *   reaches both hidden tiers; no per-tier union pass is needed. When false
+ *   (default), neither flag is forwarded — the pre-existing behavior, preserved
+ *   byte-identically for back-compat (`ExportOptions.includeSystem` alone never
+ *   reached this far; see {@link ExportOptions.includeHidden}'s JSDoc).
  */
-async function enumerateAllIndexed<T>(reader: PortableGraphReader<T>, s: ExportSelector): Promise<Set<string>> {
+async function enumerateAllIndexed<T>(
+  reader: PortableGraphReader<T>,
+  s: ExportSelector,
+  includeHidden = false
+): Promise<Set<string>> {
   const params: any = {}
   if (s.type !== undefined) params.type = s.type
   if (s.subtype !== undefined) params.subtype = s.subtype
   if (s.where !== undefined) params.where = s.where
   if (s.service !== undefined) params.service = s.service
+  if (includeHidden) {
+    params.includeInternal = true
+    params.includeSystem = true
+  }
   const ids = new Set<string>()
   let offset = 0
   // eslint-disable-next-line no-constant-condition
@@ -672,13 +720,18 @@ async function enumerateAllIndexed<T>(reader: PortableGraphReader<T>, s: ExportS
  * path does, so both paths share one predicate-evaluation code path and can only
  * disagree on candidacy, never on what a match means.
  *
- * Mirrors `find()`'s default hidden-tier policy (always hides `'internal'` and
+ * Mirrors `find()`'s hidden-tier policy given the SAME `includeSystem`/`includeHidden`
- * `'system'` here — `enumerateAllIndexed` never opts either back in via `find()`
+ * pair (see {@link enumerateAllIndexed}) so the two enumeration modes produce
- * either, since `ExportOptions.includeSystem` is applied later, per-entity, and
+ * identical id sets when the index is healthy, at ANY tier-visibility setting.
- * only reachable for ids a selector already named directly) so the two
+ *
- * enumeration modes produce identical id sets when the index is healthy.
+ * @param includeSystem - The ALREADY-COMBINED `includeSystem || includeHidden` value.
+ * @param includeHidden - Whether `'internal'`-tier nouns are ALSO admitted.
  */
-async function enumerateAllCanonical(storage: StorageAdapter): Promise<Set<string>> {
+async function enumerateAllCanonical(
+  storage: StorageAdapter,
+  includeSystem = false,
+  includeHidden = false
+): Promise<Set<string>> {
   const ids = new Set<string>()
   let offset = 0
   let cursor: string | undefined
@@ -686,7 +739,8 @@ async function enumerateAllCanonical(storage: StorageAdapter): Promise<Set<strin
   while (true) {
     const page = await storage.getNouns({ pagination: { limit: ENUM_PAGE, offset, cursor } })
     for (const item of page.items) {
-      if (item.visibility === 'internal' || item.visibility === 'system') continue
+      if (item.visibility === 'internal' && !includeHidden) continue
+      if (item.visibility === 'system' && !includeSystem) continue
       ids.add(item.id)
     }
     if (!page.hasMore || page.items.length === 0) break
@@ -860,19 +914,27 @@ function toPortableGraphRelation<T>(r: Relation<T>): PortableGraphRelation {
   return br
 }
 
+/**
+ * @param includeHidden - When true, forwards `includeInternal`/`includeSystem` into
+ *   every `related()` call so hidden-tier relations reach candidacy too — mirrors
+ *   {@link enumerateAllIndexed}'s `includeHidden` handling, and preserves back-compat
+ *   when false/omitted (the pre-existing, unconditional hidden-tier exclusion).
+ */
 async function collectEdges<T>(
   reader: PortableGraphReader<T>,
   idSet: Set<string>,
-  edges: 'induced' | 'incident' | 'none'
+  edges: 'induced' | 'incident' | 'none',
+  includeHidden = false
 ): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> {
   if (edges === 'none') return { relations: [] }
 
+  const tierOptIn = includeHidden ? { includeInternal: true, includeSystem: true } : {}
   const relations: PortableGraphRelation[] = []
   const dangling = new Set<string>()
   const seen = new Set<string>()
 
   for (const id of idSet) {
-    const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT })
+    const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT, ...tierOptIn })
     for (const r of rels) {
       if (seen.has(r.id)) continue
       const toIn = idSet.has(r.to)
@@ -885,7 +947,7 @@ async function collectEdges<T>(
 
   if (edges === 'incident') {
     for (const id of idSet) {
-      const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT })
+      const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT, ...tierOptIn })
       for (const r of rels) {
         if (seen.has(r.id)) continue
         if (!idSet.has(r.from)) {
@@ -921,15 +983,21 @@ function hnswVerbToPortableGraphRelation(v: HNSWVerbWithMetadata): PortableGraph
  * branch: relations can be blinded by adjacency-index corruption regardless of
  * how `idSet` (the kept node ids) was produced.
  *
- * Mirrors `related()`'s default hidden-tier policy (always hides `'internal'`
+ * Mirrors `related()`'s default hidden-tier policy (hides `'internal'` and
- * and `'system'` — `collectEdges` never opts either back in via `related()`
+ * `'system'` unless `includeHidden`/`includeSystem` say otherwise) so the two
- * either) so the two enumeration modes produce identical relation sets when the
+ * enumeration modes produce identical relation sets when the index is healthy.
- * index is healthy.
+ *
+ * @param includeSystem - Whether `'system'`-tier verbs are admitted (the
+ *   caller passes the ALREADY-combined `includeSystem || includeHidden` value —
+ *   see `exportGraph`'s `effectiveIncludeSystem`).
+ * @param includeHidden - Whether `'internal'`-tier verbs are ALSO admitted.
  */
 async function collectEdgesCanonical(
   storage: StorageAdapter,
   idSet: Set<string>,
-  edges: 'induced' | 'incident' | 'none'
+  edges: 'induced' | 'incident' | 'none',
+  includeSystem = false,
+  includeHidden = false
 ): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> {
   if (edges === 'none') return { relations: [] }
 
@@ -944,7 +1012,8 @@ async function collectEdgesCanonical(
     const page = await storage.getVerbs({ pagination: { limit: ENUM_PAGE, offset, cursor } })
     for (const v of page.items) {
       if (seen.has(v.id)) continue
-      if (v.visibility === 'internal' || v.visibility === 'system') continue
+      if (v.visibility === 'internal' && !includeHidden) continue
+      if (v.visibility === 'system' && !includeSystem) continue
       const fromIn = idSet.has(v.sourceId)
       const toIn = idSet.has(v.targetId)
       if (edges === 'induced') {

tests/unit/db/db-portable-graph.test.ts

@@ -453,3 +453,98 @@ describe('8.0 export enumeration:"canonical" — canon-complete against index bl
     ).rejects.toThrow(/enumeration:'canonical' requires a storage adapter/)
   })
 })
+
+describe('8.0 export includeHidden — every visibility tier for migration-grade canon completeness', () => {
+  // The fixed-id VFS root Brainy.init() always creates is the one 'system'-visibility
+  // entity a consumer can rely on existing (visibility:'system' is not settable via the
+  // public add() API — "intentionally not accepted", per AddParams.visibility's doc).
+  const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000'
+
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy(createTestConfig())
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('canonical + includeHidden carries an internal row AND the system row; round-trips through import', async () => {
+    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
+    const internalId = await brain.add({
+      data: 'Internal',
+      type: NounType.Thing,
+      subtype: 'x',
+      visibility: 'internal'
+    })
+
+    const migrationExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true })
+    const ids = migrationExport.entities.map((e) => e.id)
+    expect(ids).toContain(publicId)
+    expect(ids).toContain(internalId)
+    expect(ids).toContain(VFS_ROOT_ID)
+    expect(migrationExport.entities.find((e) => e.id === internalId)?.visibility).toBe('internal')
+    expect(migrationExport.entities.find((e) => e.id === VFS_ROOT_ID)?.visibility).toBe('system')
+
+    const target = new Brainy(createTestConfig())
+    await target.init()
+    try {
+      const result = await target.import(migrationExport)
+      expect(result.errors).toHaveLength(0)
+      expect((await target.get(internalId))?.visibility).toBe('internal')
+    } finally {
+      await target.close()
+    }
+  })
+
+  it('default export (includeHidden omitted) still excludes both hidden tiers — pins today\'s behavior', async () => {
+    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
+    const internalId = await brain.add({
+      data: 'Internal',
+      type: NounType.Thing,
+      subtype: 'x',
+      visibility: 'internal'
+    })
+
+    for (const opts of [{ enumeration: 'index' as const }, { enumeration: 'canonical' as const }]) {
+      const backup = await brain.export({}, opts)
+      const ids = backup.entities.map((e) => e.id)
+      expect(ids).toContain(publicId)
+      expect(ids).not.toContain(internalId)
+      expect(ids).not.toContain(VFS_ROOT_ID)
+    }
+  })
+
+  it('index mode + includeHidden also reaches both tiers — find() takes includeInternal + includeSystem in one pass', async () => {
+    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
+    const internalId = await brain.add({
+      data: 'Internal',
+      type: NounType.Thing,
+      subtype: 'x',
+      visibility: 'internal'
+    })
+
+    const indexExport = await brain.export({}, { enumeration: 'index', includeHidden: true })
+    const canonicalExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true })
+
+    const indexIds = indexExport.entities.map((e) => e.id).sort()
+    const canonicalIds = canonicalExport.entities.map((e) => e.id).sort()
+    expect(indexIds).toEqual(canonicalIds)
+    expect(indexIds).toContain(publicId)
+    expect(indexIds).toContain(internalId)
+    expect(indexIds).toContain(VFS_ROOT_ID)
+  })
+
+  it('drift stays pure under includeHidden — no tier-policy noise when the index is healthy', async () => {
+    await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
+    await brain.add({ data: 'Internal', type: NounType.Thing, subtype: 'x', visibility: 'internal' })
+
+    const audited = await brain.export(
+      {},
+      { enumeration: 'canonical', includeHidden: true, reportIndexDrift: true }
+    )
+    expect(audited.drift).toEqual({ canonicalOnly: [], indexOnly: [] })
+  })
+})