diff --git a/RELEASES.md b/RELEASES.md index da8fa134..9e7dec6f 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -61,6 +61,15 @@ to the caller today. Migration-audit evidence, not a repair: nonzero drift is reported loudly (`console.warn` with the counts) and nothing is auto-healed — run `brain.repairIndex()` to reconcile the metadata index once drift is confirmed. +- **New: `export(selector, { includeHidden: true })`** (default: false — unchanged + behavior). Without it, a whole-brain/predicate export could never carry a + `visibility:'internal'` or `'system'` row, in EITHER `enumeration` mode — a real gap + for a bulk-migration fold auditing per-visibility-tier, where a hidden tier is real + user data, not noise to drop. `includeHidden` admits both tiers into candidacy in + both modes (and implies `includeSystem`; `includeSystem` alone keeps its narrower, + pre-existing meaning). **Migration-grade exports set `includeHidden: true`** — a + complete-canon export must carry every visibility tier; consumer-facing exports + leave it off. - **Ops note (consumer-invisible): the release pipeline's forge-registry publish now runs on CI**, triggered by the release tag, instead of PUTting the tarball from the laptop over WAN — no change to what gets published or how a consumer installs it. diff --git a/src/db/db.ts b/src/db/db.ts index ca9c133b..c5cbad8b 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -528,6 +528,11 @@ export class Db { * throws {@link CanonicalEnumerationUnavailableError} rather than silently mixing * generations or missing the overlay's own entities. * + * `options.includeHidden: true` admits BOTH hidden visibility tiers + * (`'internal'` and `'system'`) into a whole-brain/predicate export, in EITHER + * `enumeration` mode — see {@link ExportOptions.includeHidden}. Migration-grade + * exports set this; consumer-facing exports leave it off (default: false). + * * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}. * @param options - HOW to export (vectors / VFS bytes / edge policy / enumeration mode). See {@link ExportOptions}. * @returns A versioned, portable `PortableGraph` document. diff --git a/src/db/portableGraph.ts b/src/db/portableGraph.ts index 4c50ef5b..f3134325 100644 --- a/src/db/portableGraph.ts +++ b/src/db/portableGraph.ts @@ -94,6 +94,22 @@ export interface ExportOptions { includeContent?: boolean /** Include `visibility:'system'` entities (e.g. the VFS root) (default: false). */ includeSystem?: boolean + /** + * Admit BOTH hidden visibility tiers — `'internal'` AND `'system'` — into the + * whole-brain/predicate candidate set, in EITHER `enumeration` mode (default: + * false — today's behavior is byte-identical). `includeSystem` alone only ever + * reached `'system'` for a structural selector's per-entity gate; whole-brain/ + * predicate enumeration never forwarded it into the candidate walk at all, so a + * hidden-tier row could never survive a whole-brain export regardless of any + * flag — the gap this option closes. `includeHidden: true` IMPLIES + * `includeSystem: true` (both tiers are admitted together; there is no + * "system but not internal" combination via this flag) — `includeSystem` on + * its own keeps its narrower, pre-existing meaning for back-compat. + * + * Migration-grade exports set `includeHidden: true` — a complete-canon export + * must carry every visibility tier; consumer-facing exports leave it off. + */ + includeHidden?: boolean /** Which edges to include (default: `'induced'`). */ edges?: 'induced' | 'incident' | 'none' /** @@ -358,6 +374,7 @@ export async function exportGraph( includeVectors = false, includeContent = false, includeSystem = false, + includeHidden = false, edges = 'induced', enumeration = 'index', reportIndexDrift = false @@ -371,15 +388,23 @@ export async function exportGraph( ) } const wantDrift = enumeration === 'canonical' && reportIndexDrift + // includeHidden IMPLIES includeSystem (see ExportOptions.includeHidden's JSDoc) — every + // system-tier gate below reads THIS combined value, never the raw option, so + // `includeHidden` alone is always sufficient to see system-tier rows too. + const effectiveIncludeSystem = includeSystem || includeHidden // 1. Resolve the node-id set (+ the index's raw candidate set, only when diffing it). + // Both `enumerateAllCanonical` and `enumerateAllIndexed` receive the SAME + // `effectiveIncludeSystem`/`includeHidden` pair below, so a drift diff can never + // contain tier-policy noise — only genuine index-vs-canonical disagreement. const { idSet, indexCandidateIds } = await resolveSelector( reader, storage, selector, - includeSystem, + effectiveIncludeSystem, enumeration, - wantDrift + wantDrift, + includeHidden ) // 2. Read canonical entities (reserved fields top-level), applying any predicate filter. @@ -392,7 +417,7 @@ export async function exportGraph( for (const id of idSet) { const e = await reader.get(id, { includeVectors }) if (!e) continue - if (!includeSystem && (e as any).visibility === 'system') continue + if (!effectiveIncludeSystem && (e as any).visibility === 'system') continue if (usePredicate && !matchesPredicate(e, selector)) continue entityMap.set(id, e) entities.push(toPortableGraphEntity(e, includeVectors)) @@ -422,8 +447,8 @@ export async function exportGraph( // relation regardless of how the node set was produced. const { relations, danglingIds } = enumeration === 'canonical' - ? await collectEdgesCanonical(storage!, keptIds, edges) - : await collectEdges(reader, keptIds, edges) + ? await collectEdgesCanonical(storage!, keptIds, edges, effectiveIncludeSystem, includeHidden) + : await collectEdges(reader, keptIds, edges, includeHidden) // 4. VFS blob bytes (only when requested). let blobs: Record | undefined @@ -598,7 +623,9 @@ function hasPredicate(s: ExportSelector): boolean { * @param storage - Storage adapter (only touched when `enumeration:'canonical'` * resolves the whole-brain/predicate branch). * @param s - The export selector. - * @param includeSystem - Whether `visibility:'system'` entities are wanted. + * @param includeSystem - The ALREADY-COMBINED `includeSystem || includeHidden` value + * (see `exportGraph`'s `effectiveIncludeSystem`) — whether `visibility:'system'` + * entities are wanted. * @param enumeration - `'index'` (default) or `'canonical'` — see {@link ExportOptions.enumeration}. * Only affects the whole-brain/predicate branch (the `else` below): structural * selectors (`ids`/`collection`/`connected`/`vfsPath`) never rode the metadata @@ -606,6 +633,9 @@ function hasPredicate(s: ExportSelector): boolean { * @param wantIndexCandidates - When true (only meaningful with `enumeration:'canonical'` * on the whole-brain/predicate branch), ALSO run the index-based walk and return * its raw candidate set as `indexCandidateIds`, for {@link ExportIndexDrift}. + * @param includeHidden - Whether `visibility:'internal'` entities are ALSO wanted + * (see {@link ExportOptions.includeHidden}). Threaded to BOTH enumeration + * functions identically so a drift diff never contains tier-policy noise. */ async function resolveSelector( reader: PortableGraphReader, @@ -613,7 +643,8 @@ async function resolveSelector( s: ExportSelector, includeSystem: boolean, enumeration: 'index' | 'canonical', - wantIndexCandidates: boolean + wantIndexCandidates: boolean, + includeHidden: boolean ): Promise<{ idSet: Set; indexCandidateIds?: Set }> { let idSet: Set let indexCandidateIds: Set | undefined @@ -626,10 +657,10 @@ async function resolveSelector( } else if (s.vfsPath) { idSet = await resolveVfsPath(reader, s.vfsPath, s.recursive ?? true, s.depth) } else if (enumeration === 'canonical') { - idSet = await enumerateAllCanonical(storage!) - if (wantIndexCandidates) indexCandidateIds = await enumerateAllIndexed(reader, s) + idSet = await enumerateAllCanonical(storage!, includeSystem, includeHidden) + if (wantIndexCandidates) indexCandidateIds = await enumerateAllIndexed(reader, s, includeHidden) } else { - idSet = await enumerateAllIndexed(reader, s) + idSet = await enumerateAllIndexed(reader, s, includeHidden) } if (!includeSystem) idSet.delete(VFS_ROOT_ID) return { idSet, indexCandidateIds } @@ -640,13 +671,30 @@ async function resolveSelector( * paginated `find()`. The metadata index is an acceleration structure over * this candidate set — see {@link enumerateAllCanonical} for the storage-level * counterpart that never consults it. + * + * @param includeHidden - When true, forwards `includeInternal: true` AND + * `includeSystem: true` into the SAME `find()` call — `find()` supports both + * flags simultaneously (confirmed via `FindParams.includeInternal`/`includeSystem` + * and `Brainy`'s `resolveHiddenIds`/`excludedVisibilityTiers`), so ONE pass + * reaches both hidden tiers; no per-tier union pass is needed. When false + * (default), neither flag is forwarded — the pre-existing behavior, preserved + * byte-identically for back-compat (`ExportOptions.includeSystem` alone never + * reached this far; see {@link ExportOptions.includeHidden}'s JSDoc). */ -async function enumerateAllIndexed(reader: PortableGraphReader, s: ExportSelector): Promise> { +async function enumerateAllIndexed( + reader: PortableGraphReader, + s: ExportSelector, + includeHidden = false +): Promise> { const params: any = {} if (s.type !== undefined) params.type = s.type if (s.subtype !== undefined) params.subtype = s.subtype if (s.where !== undefined) params.where = s.where if (s.service !== undefined) params.service = s.service + if (includeHidden) { + params.includeInternal = true + params.includeSystem = true + } const ids = new Set() let offset = 0 // eslint-disable-next-line no-constant-condition @@ -672,13 +720,18 @@ async function enumerateAllIndexed(reader: PortableGraphReader, s: ExportS * path does, so both paths share one predicate-evaluation code path and can only * disagree on candidacy, never on what a match means. * - * Mirrors `find()`'s default hidden-tier policy (always hides `'internal'` and - * `'system'` here — `enumerateAllIndexed` never opts either back in via `find()` - * either, since `ExportOptions.includeSystem` is applied later, per-entity, and - * only reachable for ids a selector already named directly) so the two - * enumeration modes produce identical id sets when the index is healthy. + * Mirrors `find()`'s hidden-tier policy given the SAME `includeSystem`/`includeHidden` + * pair (see {@link enumerateAllIndexed}) so the two enumeration modes produce + * identical id sets when the index is healthy, at ANY tier-visibility setting. + * + * @param includeSystem - The ALREADY-COMBINED `includeSystem || includeHidden` value. + * @param includeHidden - Whether `'internal'`-tier nouns are ALSO admitted. */ -async function enumerateAllCanonical(storage: StorageAdapter): Promise> { +async function enumerateAllCanonical( + storage: StorageAdapter, + includeSystem = false, + includeHidden = false +): Promise> { const ids = new Set() let offset = 0 let cursor: string | undefined @@ -686,7 +739,8 @@ async function enumerateAllCanonical(storage: StorageAdapter): Promise(r: Relation): PortableGraphRelation { return br } +/** + * @param includeHidden - When true, forwards `includeInternal`/`includeSystem` into + * every `related()` call so hidden-tier relations reach candidacy too — mirrors + * {@link enumerateAllIndexed}'s `includeHidden` handling, and preserves back-compat + * when false/omitted (the pre-existing, unconditional hidden-tier exclusion). + */ async function collectEdges( reader: PortableGraphReader, idSet: Set, - edges: 'induced' | 'incident' | 'none' + edges: 'induced' | 'incident' | 'none', + includeHidden = false ): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> { if (edges === 'none') return { relations: [] } + const tierOptIn = includeHidden ? { includeInternal: true, includeSystem: true } : {} const relations: PortableGraphRelation[] = [] const dangling = new Set() const seen = new Set() for (const id of idSet) { - const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT }) + const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT, ...tierOptIn }) for (const r of rels) { if (seen.has(r.id)) continue const toIn = idSet.has(r.to) @@ -885,7 +947,7 @@ async function collectEdges( if (edges === 'incident') { for (const id of idSet) { - const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT }) + const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT, ...tierOptIn }) for (const r of rels) { if (seen.has(r.id)) continue if (!idSet.has(r.from)) { @@ -921,15 +983,21 @@ function hnswVerbToPortableGraphRelation(v: HNSWVerbWithMetadata): PortableGraph * branch: relations can be blinded by adjacency-index corruption regardless of * how `idSet` (the kept node ids) was produced. * - * Mirrors `related()`'s default hidden-tier policy (always hides `'internal'` - * and `'system'` — `collectEdges` never opts either back in via `related()` - * either) so the two enumeration modes produce identical relation sets when the - * index is healthy. + * Mirrors `related()`'s default hidden-tier policy (hides `'internal'` and + * `'system'` unless `includeHidden`/`includeSystem` say otherwise) so the two + * enumeration modes produce identical relation sets when the index is healthy. + * + * @param includeSystem - Whether `'system'`-tier verbs are admitted (the + * caller passes the ALREADY-combined `includeSystem || includeHidden` value — + * see `exportGraph`'s `effectiveIncludeSystem`). + * @param includeHidden - Whether `'internal'`-tier verbs are ALSO admitted. */ async function collectEdgesCanonical( storage: StorageAdapter, idSet: Set, - edges: 'induced' | 'incident' | 'none' + edges: 'induced' | 'incident' | 'none', + includeSystem = false, + includeHidden = false ): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> { if (edges === 'none') return { relations: [] } @@ -944,7 +1012,8 @@ async function collectEdgesCanonical( const page = await storage.getVerbs({ pagination: { limit: ENUM_PAGE, offset, cursor } }) for (const v of page.items) { if (seen.has(v.id)) continue - if (v.visibility === 'internal' || v.visibility === 'system') continue + if (v.visibility === 'internal' && !includeHidden) continue + if (v.visibility === 'system' && !includeSystem) continue const fromIn = idSet.has(v.sourceId) const toIn = idSet.has(v.targetId) if (edges === 'induced') { diff --git a/tests/unit/db/db-portable-graph.test.ts b/tests/unit/db/db-portable-graph.test.ts index d70836b5..1de9983d 100644 --- a/tests/unit/db/db-portable-graph.test.ts +++ b/tests/unit/db/db-portable-graph.test.ts @@ -453,3 +453,98 @@ describe('8.0 export enumeration:"canonical" — canon-complete against index bl ).rejects.toThrow(/enumeration:'canonical' requires a storage adapter/) }) }) + +describe('8.0 export includeHidden — every visibility tier for migration-grade canon completeness', () => { + // The fixed-id VFS root Brainy.init() always creates is the one 'system'-visibility + // entity a consumer can rely on existing (visibility:'system' is not settable via the + // public add() API — "intentionally not accepted", per AddParams.visibility's doc). + const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('canonical + includeHidden carries an internal row AND the system row; round-trips through import', async () => { + const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' }) + const internalId = await brain.add({ + data: 'Internal', + type: NounType.Thing, + subtype: 'x', + visibility: 'internal' + }) + + const migrationExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true }) + const ids = migrationExport.entities.map((e) => e.id) + expect(ids).toContain(publicId) + expect(ids).toContain(internalId) + expect(ids).toContain(VFS_ROOT_ID) + expect(migrationExport.entities.find((e) => e.id === internalId)?.visibility).toBe('internal') + expect(migrationExport.entities.find((e) => e.id === VFS_ROOT_ID)?.visibility).toBe('system') + + const target = new Brainy(createTestConfig()) + await target.init() + try { + const result = await target.import(migrationExport) + expect(result.errors).toHaveLength(0) + expect((await target.get(internalId))?.visibility).toBe('internal') + } finally { + await target.close() + } + }) + + it('default export (includeHidden omitted) still excludes both hidden tiers — pins today\'s behavior', async () => { + const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' }) + const internalId = await brain.add({ + data: 'Internal', + type: NounType.Thing, + subtype: 'x', + visibility: 'internal' + }) + + for (const opts of [{ enumeration: 'index' as const }, { enumeration: 'canonical' as const }]) { + const backup = await brain.export({}, opts) + const ids = backup.entities.map((e) => e.id) + expect(ids).toContain(publicId) + expect(ids).not.toContain(internalId) + expect(ids).not.toContain(VFS_ROOT_ID) + } + }) + + it('index mode + includeHidden also reaches both tiers — find() takes includeInternal + includeSystem in one pass', async () => { + const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' }) + const internalId = await brain.add({ + data: 'Internal', + type: NounType.Thing, + subtype: 'x', + visibility: 'internal' + }) + + const indexExport = await brain.export({}, { enumeration: 'index', includeHidden: true }) + const canonicalExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true }) + + const indexIds = indexExport.entities.map((e) => e.id).sort() + const canonicalIds = canonicalExport.entities.map((e) => e.id).sort() + expect(indexIds).toEqual(canonicalIds) + expect(indexIds).toContain(publicId) + expect(indexIds).toContain(internalId) + expect(indexIds).toContain(VFS_ROOT_ID) + }) + + it('drift stays pure under includeHidden — no tier-policy noise when the index is healthy', async () => { + await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' }) + await brain.add({ data: 'Internal', type: NounType.Thing, subtype: 'x', visibility: 'internal' }) + + const audited = await brain.export( + {}, + { enumeration: 'canonical', includeHidden: true, reportIndexDrift: true } + ) + expect(audited.drift).toEqual({ canonicalOnly: [], indexOnly: [] }) + }) +})