diff --git a/.forgejo/workflows/publish-forge.yml b/.forgejo/workflows/publish-forge.yml new file mode 100644 index 00000000..fb7428bf --- /dev/null +++ b/.forgejo/workflows/publish-forge.yml @@ -0,0 +1,67 @@ +name: Publish (forge) + +# Datacenter-side forge publish, moved off the laptop: an 87MB tarball PUT +# over the laptop's WAN times out; the forge's own runner does it in seconds. +# scripts/release.sh tags + pushes, then polls this workflow's result (npm +# view against the forge registry) before it ever touches the npmjs leg — +# see the "delegation contract" in scripts/release.sh's forge-publish step. + +on: + push: + tags: + - 'v*' + +jobs: + publish: + name: Publish to the forge registry + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - run: npm ci + - run: npm run build + - name: Publish + readback-verify on the forge registry + env: + FORGE_NPM_TOKEN: ${{ secrets.FORGE_NPM_TOKEN }} + run: | + set -eo pipefail + + FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" + VERSION="$(node -p "require('./package.json').version")" + echo "Publishing @soulcraft/brainy@${VERSION} to the forge registry..." + + TMPRC="$(mktemp)" + chmod 600 "$TMPRC" + { + echo "@soulcraft:registry=${FORGE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}" + } > "$TMPRC" + + # The release script bumps package.json's version before it tags, so + # this tag's checkout already carries the version being published — + # nothing here re-derives it from the tag name. + PUBLISH_OK=true + if ! npm publish --tag latest --userconfig "$TMPRC"; then + PUBLISH_OK=false + fi + + # Readback verify is the source of truth, run regardless of the publish + # exit code: a benign duplicate publish (a prior run, or a mirror, already + # landed this exact version) reports failure even though the registry + # already holds the right content. + LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" + rm -f "$TMPRC" + + if [ "$LANDED_VERSION" != "$VERSION" ]; then + echo "::error::Readback verify FAILED — the forge registry reports version '${LANDED_VERSION:-}', expected '${VERSION}'. This is a genuine publish failure, not a benign duplicate." + exit 1 + fi + + if [ "$PUBLISH_OK" = true ]; then + echo "Published and verified @soulcraft/brainy@${VERSION} on the forge registry." + else + echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on the forge (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index a5f344cc..04283b67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27) + +- docs: the last two archived-host links point home (91ef1c8b) +- feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) +- feat(release): the forge publish leg moves to CI on the tag push; the laptop verifies by readback and keeps the abort-before-storefront guard (3e4a17dc) +- feat: canonical enumeration mode for export — storage-walked, canon-complete, with an index-drift report (4d196af4) +- ci: run the pipeline on the forge (999d0ebb) + + ### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) diff --git a/README.md b/README.md index 2caf6493..ca558340 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- Brainy + Brainy

Brainy

diff --git a/RELEASES.md b/RELEASES.md index 03d7283a..2e137e4f 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,7 +1,7 @@ # @soulcraft/brainy — Release Notes for Consumers This file is the **quick reference for downstream sessions** tracking Brainy changes. -Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/soulcraftlabs/brainy/releases +Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases **How to use:** Brainy is the underlying data engine for downstream applications. Read this when: - Upgrading `@soulcraft/brainy` in your application @@ -31,6 +31,49 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## Unreleased (canonical enumeration mode for export — storage-walked, canon-complete) + +From a fleet data-migration program's requirement for whole-brain exports that are +provably canon-complete: `export()`'s default enumeration for a whole-brain/predicate +selector is a generation-correct paginated `find()` walk — a projection query riding +the metadata index as an acceleration structure. Production has documented both of the +index's failure classes: a lost/stale posting can silently OMIT a canonical record from +an export, and a stale posting can silently INCLUDE a phantom row. Neither is visible +to the caller today. + +- **New: `export(selector, { enumeration: 'canonical' })`** (default remains `'index'` — + unchanged behavior on this release). Canonical mode walks every live noun/verb + directly off the storage adapter's canonical shard layout (`storage.getNouns()` / + `getVerbs()` — the same primitive `repairIndex()`'s recount and every index-heal + walk use) instead of the metadata/graph indexes, then applies the selector as a + plain predicate over the walked records. This guarantees canon-completeness — index + corruption cannot hide a live record from the export — at the cost of an O(N) walk + regardless of selector selectivity. Relations are also walked canonically in this + mode, for every selector, not just the whole-brain case. Requires the LIVE current + generation: called on a historical `asOf()` view or a speculative `with()` overlay it + throws `CanonicalEnumerationUnavailableError` rather than silently mixing generations + or missing an overlay's own entities — `enumeration: 'index'` (the default) is + unaffected and still composes with `asOf()`/`with()` as before. +- **New: `export(selector, { enumeration: 'canonical', reportIndexDrift: true })`** — + also runs the index-based enumeration and diffs it against canonical ground truth, + attaching `PortableGraph.drift: { canonicalOnly: string[], indexOnly: string[] }` + (canon-present ids the index missed; index-visible ids canon-absent — phantoms). + Migration-audit evidence, not a repair: nonzero drift is reported loudly + (`console.warn` with the counts) and nothing is auto-healed — run `brain.repairIndex()` + to reconcile the metadata index once drift is confirmed. +- **New: `export(selector, { includeHidden: true })`** (default: false — unchanged + behavior). Without it, a whole-brain/predicate export could never carry a + `visibility:'internal'` or `'system'` row, in EITHER `enumeration` mode — a real gap + for a bulk-migration fold auditing per-visibility-tier, where a hidden tier is real + user data, not noise to drop. `includeHidden` admits both tiers into candidacy in + both modes (and implies `includeSystem`; `includeSystem` alone keeps its narrower, + pre-existing meaning). **Migration-grade exports set `includeHidden: true`** — a + complete-canon export must carry every visibility tier; consumer-facing exports + leave it off. +- **Ops note (consumer-invisible): the release pipeline's forge-registry publish now runs + on CI**, triggered by the release tag, instead of PUTting the tarball from the laptop + over WAN — no change to what gets published or how a consumer installs it. + ## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers) From a production incident: a native-provider op ground 38-40s inside a transaction, diff --git a/package-lock.json b/package-lock.json index d0c7b9d9..29be914a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.11.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index ce670369..cfb05486 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.10.1", + "version": "8.11.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/release.sh b/scripts/release.sh index 43fa50bd..c64d6c5e 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -181,30 +181,33 @@ echo -e "${BLUE}8️⃣ Pushing to origin...${NC}" git push --follow-tags origin "$CURRENT_BRANCH" echo -e "${GREEN}✅ Pushed to origin${NC}\n" -# Step 10: Publish — forge FIRST (home), npmjs second (the world's storefront). -# The fleet-wide ~/.npmrc maps the @soulcraft scope to the forge registry, and -# a scope mapping BEATS `--registry` on the command line — so each publish -# names its registry via the scope override explicitly. Nothing implicit. +# Step 10: Forge publish is CI's job now, not the laptop's — a tag push (just +# above) triggers .forgejo/workflows/publish-forge.yml, which builds and +# publishes on the forge's own runner (datacenter-side: seconds, not the +# laptop's WAN timing out on an 87MB tarball PUT). The laptop holds no forge +# publish credential anymore; it only waits for CI's result before trusting +# the forge/npmjs pair enough to publish the storefront leg. FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" -FORGE_NPM_TOKEN_FILE="$HOME/.config/soulcraft/npm-publish-brainy.token" -echo -e "${BLUE}9️⃣ Publishing to the forge registry (home)...${NC}" -if [ -f "$FORGE_NPM_TOKEN_FILE" ]; then - TMPRC="$(mktemp)" - chmod 600 "$TMPRC" - { - echo "@soulcraft:registry=${FORGE_NPM_REG}" - echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=$(cat "$FORGE_NPM_TOKEN_FILE")" - } > "$TMPRC" - if npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then - echo -e "${GREEN}✅ Published to the forge${NC}\n" - else - rm -f "$TMPRC" - echo -e "${RED}❌ Forge publish FAILED — aborting before npmjs so the pair never diverges. Fix and re-run.${NC}" - exit 1 +FORGE_POLL_INTERVAL_S=15 +FORGE_POLL_MAX_ATTEMPTS=40 # 40 × 15s = 10 minutes +echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to the forge registry (home)...${NC}" +FORGE_LANDED=false +for ((attempt = 1; attempt <= FORGE_POLL_MAX_ATTEMPTS; attempt++)); do + LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${FORGE_NPM_REG}" 2>/dev/null || echo "") + if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then + FORGE_LANDED=true + break fi - rm -f "$TMPRC" + echo -e "${YELLOW} … not yet on the forge (attempt ${attempt}/${FORGE_POLL_MAX_ATTEMPTS}); retrying in ${FORGE_POLL_INTERVAL_S}s${NC}" + sleep "$FORGE_POLL_INTERVAL_S" +done + +if [ "$FORGE_LANDED" = true ]; then + echo -e "${GREEN}✅ CI published v${NEW_VERSION} to the forge${NC}\n" else - echo -e "${RED}❌ Forge publish token missing (${FORGE_NPM_TOKEN_FILE}) — aborting. The forge is home; publish it first or restage the token.${NC}" + echo -e "${RED}❌ CI forge publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}" + echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}" + echo -e "${RED} forge registry after ${FORGE_POLL_MAX_ATTEMPTS} attempts, ${FORGE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}" exit 1 fi diff --git a/src/db/db.ts b/src/db/db.ts index ac927fc5..c5cbad8b 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -66,7 +66,7 @@ import { import { v4 as uuidv4 } from '../universal/uuid.js' import { coerceNewEntityId, resolveEntityId, ORIGINAL_ID_KEY } from '../utils/idNormalization.js' import { EntityNotFoundError } from '../errors/notFound.js' -import { SpeculativeOverlayError } from './errors.js' +import { SpeculativeOverlayError, CanonicalEnumerationUnavailableError } from './errors.js' import type { GenerationStore } from './generationStore.js' import type { ChangedIds, TransactReceipt, TxOperation } from './types.js' import { entityMatchesFind, resolveEntityField, UnsupportedWhereOperatorError } from './whereMatcher.js' @@ -520,14 +520,37 @@ export class Db { * (no generation history) — distinct from `persist()` (native whole-brain snapshot * that preserves history). Restore with `brain.import(backup)`. * + * `options.enumeration: 'canonical'` (default: `'index'`) walks the storage + * adapter's canonical noun/verb layout directly instead of the metadata/graph + * indexes, guaranteeing canon-completeness against index corruption — see + * {@link ExportOptions.enumeration}. It requires the LIVE, current-generation + * view: called on a historical `asOf()` pin or a speculative `with()` overlay it + * throws {@link CanonicalEnumerationUnavailableError} rather than silently mixing + * generations or missing the overlay's own entities. + * + * `options.includeHidden: true` admits BOTH hidden visibility tiers + * (`'internal'` and `'system'`) into a whole-brain/predicate export, in EITHER + * `enumeration` mode — see {@link ExportOptions.includeHidden}. Migration-grade + * exports set this; consumer-facing exports leave it off (default: false). + * * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}. - * @param options - HOW to export (vectors / VFS bytes / edge policy). See {@link ExportOptions}. + * @param options - HOW to export (vectors / VFS bytes / edge policy / enumeration mode). See {@link ExportOptions}. * @returns A versioned, portable `PortableGraph` document. + * @throws {@link CanonicalEnumerationUnavailableError} if `enumeration:'canonical'` is + * requested on a historical or speculative-overlay view. * @example * const backup = await brain.now().export({ collection: id }, { includeVectors: true }) + * @example + * // Canon-complete audit export, with an index-drift report attached. + * const audit = await brain.now().export({}, { enumeration: 'canonical', reportIndexDrift: true }) + * if (audit.drift) console.log(audit.drift.canonicalOnly, audit.drift.indexOnly) */ async export(selector: ExportSelector = {}, options: ExportOptions = {}): Promise { this.assertUsable('export') + if (options.enumeration === 'canonical') { + if (this.overlay) throw new CanonicalEnumerationUnavailableError(this.gen, 'overlay') + if (this.isHistorical()) throw new CanonicalEnumerationUnavailableError(this.gen, 'historical') + } return exportGraph(this, this.host.storage, selector, options) } diff --git a/src/db/errors.ts b/src/db/errors.ts index 22f405be..e20488f8 100644 --- a/src/db/errors.ts +++ b/src/db/errors.ts @@ -23,8 +23,12 @@ * serve the full query surface via at-generation index materialization. * - {@link GenerationCompactedError} — `asOf()` asked for a generation whose * immutable records were reclaimed by `compactHistory()`. + * - {@link CanonicalEnumerationUnavailableError} — `export()`'s + * `enumeration:'canonical'` mode was called on a historical `asOf()` view or a + * speculative `with()` overlay; the canonical storage walk only ever answers + * "what is live right now." * - * All three are exported from the package root (`@soulcraft/brainy`). + * All are exported from the package root (`@soulcraft/brainy`). */ /** @@ -160,6 +164,64 @@ export class GenerationCompactedError extends Error { } } +/** + * @description Thrown by `db.export(selector, { enumeration: 'canonical' })` when + * the `Db` it is called on is not the live, current-generation view: a historical + * `brain.asOf(g)` pin, or a speculative `db.with()` overlay. + * + * Canonical enumeration mode walks the storage adapter's canonical shard layout + * directly (`storage.getNouns()`/`getVerbs()`) instead of the metadata/graph + * indexes — but that walk has no generation parameter, it can only ever answer + * "what is live right now." Serving it against a historical pin would silently + * mix generations (today's canonical records under yesterday's selector), and + * against a speculative overlay it would silently miss the overlay's own + * in-memory entities (which never touched storage). Both are exactly the kind of + * silently-wrong result canonical mode exists to prevent elsewhere — so this + * boundary throws instead. + * + * `enumeration: 'index'` (the default) is unaffected: it composes with + * `asOf()`/`with()` exactly as before, via the generation-correct `find()` walk. + * + * @example + * const past = await brain.asOf(g1) + * try { + * await past.export({}, { enumeration: 'canonical' }) + * } catch (err) { + * if (err instanceof CanonicalEnumerationUnavailableError) { + * // Time-travel export: use the default index-based enumeration instead. + * await past.export({}, { enumeration: 'index' }) + * } + * } + */ +export class CanonicalEnumerationUnavailableError extends Error { + /** The view's pinned generation. */ + public readonly generation: number + /** Why canonical mode cannot serve this view. */ + public readonly reason: 'historical' | 'overlay' + + /** + * @param generation - The view's pinned generation. + * @param reason - `'historical'` (a past `asOf()` pin) or `'overlay'` (a speculative `with()`). + */ + constructor(generation: number, reason: 'historical' | 'overlay') { + const what = + reason === 'historical' + ? `a historical view pinned at generation ${generation}` + : `a speculative with() overlay (base generation ${generation})` + super( + `export()'s enumeration:'canonical' requires the live, current-generation view — ` + + `it was called on ${what}. The canonical storage walk has no generation parameter, ` + + `so it can only answer "what is live right now"; serving it here would silently ` + + `mix generations (historical) or miss the overlay's own in-memory entities ` + + `(overlay). Use enumeration:'index' (the default) for a time-travel or what-if ` + + `export, or pin brain.now() for a live canonical export.` + ) + this.name = 'CanonicalEnumerationUnavailableError' + this.generation = generation + this.reason = reason + } +} + /** One entity/relationship left in an unreconciled state by a failed rollback. */ export interface UnreconciledRecord { /** The entity or relationship id. */ diff --git a/src/db/portableGraph.ts b/src/db/portableGraph.ts index d8f57b78..f3134325 100644 --- a/src/db/portableGraph.ts +++ b/src/db/portableGraph.ts @@ -27,7 +27,7 @@ import { Entity, Relation, Result } from '../types/brainy.types.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { StorageAdapter } from '../coreTypes.js' +import { StorageAdapter, HNSWVerbWithMetadata } from '../coreTypes.js' import { getBrainyVersion } from '../utils/version.js' import { TxOperation } from './types.js' @@ -94,8 +94,85 @@ export interface ExportOptions { includeContent?: boolean /** Include `visibility:'system'` entities (e.g. the VFS root) (default: false). */ includeSystem?: boolean + /** + * Admit BOTH hidden visibility tiers — `'internal'` AND `'system'` — into the + * whole-brain/predicate candidate set, in EITHER `enumeration` mode (default: + * false — today's behavior is byte-identical). `includeSystem` alone only ever + * reached `'system'` for a structural selector's per-entity gate; whole-brain/ + * predicate enumeration never forwarded it into the candidate walk at all, so a + * hidden-tier row could never survive a whole-brain export regardless of any + * flag — the gap this option closes. `includeHidden: true` IMPLIES + * `includeSystem: true` (both tiers are admitted together; there is no + * "system but not internal" combination via this flag) — `includeSystem` on + * its own keeps its narrower, pre-existing meaning for back-compat. + * + * Migration-grade exports set `includeHidden: true` — a complete-canon export + * must carry every visibility tier; consumer-facing exports leave it off. + */ + includeHidden?: boolean /** Which edges to include (default: `'induced'`). */ edges?: 'induced' | 'incident' | 'none' + /** + * How the whole-brain / predicate selector (no `ids`/`collection`/`connected`/ + * `vfsPath`) resolves its candidate id set: + * + * - `'index'` (DEFAULT — unchanged behavior) — the generation-correct + * paginated `find()` walk. Fast (O(matches), not O(N)) but rides the + * metadata index as an acceleration structure: a lost/stale index posting + * can silently OMIT a canonical record, and a stale posting pointing at a + * record that no longer matches can silently produce a phantom (dropped + * later by the same predicate re-check `'canonical'` mode also runs, so + * phantoms never reach `entities` — but they ARE lost silently unless + * {@link reportIndexDrift} is set). + * - `'canonical'` — walks every live noun/verb directly off the storage + * adapter's canonical shard layout (`storage.getNouns()`/`getVerbs()` — + * the same primitive `repairIndex()`'s recount and every index-heal walk + * use), then applies the selector as a plain predicate over the walked + * records. This GUARANTEES canon-completeness — the metadata/graph + * indexes are never consulted, so their corruption cannot hide a live + * record — at the cost of an O(N) walk regardless of selector + * selectivity (unlike the index path's O(matches)). Structural selectors + * (`ids`/`collection`/`connected`/`vfsPath`) resolve their node set + * exactly as in `'index'` mode either way (they never rode the metadata + * index); `'canonical'` additionally walks relations canonically for + * EVERY selector, since a lost adjacency-index posting can hide a + * relation regardless of how the node set was produced. Requires a + * storage adapter and the CURRENT generation — throws on a historical + * `asOf()` view or a speculative `with()` overlay (see + * {@link CanonicalEnumerationUnavailableError}), because the canonical + * walk has no notion of "as of a past generation." + */ + enumeration?: 'index' | 'canonical' + /** + * Only meaningful with `enumeration:'canonical'` (ignored otherwise): ALSO + * run the `'index'` enumeration in parallel and diff it against the + * canonical ground truth, attaching the result as {@link PortableGraph.drift}. + * Never auto-heals anything — this is migration-audit evidence, reported + * loudly (`console.warn` with the counts) whenever either list is non-empty, + * never silently. Default: false. + */ + reportIndexDrift?: boolean +} + +/** + * @description Index-vs-canonical drift for one `export({ enumeration: 'canonical', + * reportIndexDrift: true })` call. Only populated for the whole-brain / predicate + * selector (structural selectors never consulted the metadata index for their node + * set, so there is nothing to diff — both lists are empty for those). + */ +export interface ExportIndexDrift { + /** + * Ids the canonical storage walk confirmed (live, selector-matching) that the + * index-based `find()` enumeration did NOT return — canonical records the + * metadata index has lost track of. + */ + canonicalOnly: string[] + /** + * Ids the index-based `find()` enumeration returned for this selector that + * canonical ground truth (the storage walk + the same predicate check) does + * NOT support — phantom index rows (stale or cross-bucket postings). + */ + indexOnly: string[] } /** Controls how a `PortableGraph` is applied on `import()`. */ @@ -151,6 +228,8 @@ export interface PortableGraph { relations: PortableGraphRelation[] blobs?: Record danglingIds?: string[] + /** Present only when `export()` was called with `reportIndexDrift: true`. */ + drift?: ExportIndexDrift stats: { entityCount: number; relationCount: number; blobCount: number; vectorDimensions?: number } } @@ -271,11 +350,19 @@ export function validatePortableGraph(data: unknown): PortableGraphValidation { /** * @description Serialize part or all of a graph (read through `reader` at its pinned * generation) into a portable `PortableGraph` document. + * + * `enumeration:'canonical'` (see {@link ExportOptions.enumeration}) requires a + * storage adapter and the current generation: it throws + * {@link CanonicalEnumerationUnavailableError} if `storage` is absent, and the + * caller (`Db.export()`) throws the same error before this runs if the view is + * historical or a speculative overlay — the canonical storage walk has no + * generation parameter, so it can only ever answer "as of right now." + * * @param reader - Generation-correct read surface (`Db` or `Brainy`). - * @param storage - Storage adapter (used only for VFS blob bytes when `includeContent`). + * @param storage - Storage adapter (VFS blob bytes when `includeContent`; the + * canonical noun/verb walk when `enumeration:'canonical'`). * @param selector - WHAT to export (omit for the whole brain). - * @param options - HOW to export (vectors / file bytes / edge policy). - * @param dimensions - Embedding dimensionality for the manifest. + * @param options - HOW to export (vectors / file bytes / edge policy / enumeration mode). */ export async function exportGraph( reader: PortableGraphReader, @@ -287,28 +374,81 @@ export async function exportGraph( includeVectors = false, includeContent = false, includeSystem = false, - edges = 'induced' + includeHidden = false, + edges = 'induced', + enumeration = 'index', + reportIndexDrift = false } = options - // 1. Resolve the node-id set. - const idSet = await resolveSelector(reader, selector, includeSystem) + if (enumeration === 'canonical' && !storage) { + throw new Error( + `export(): enumeration:'canonical' requires a storage adapter, but none was supplied ` + + `to this reader. Use enumeration:'index' (the default), or export through a Db/Brainy ` + + `that carries its storage adapter.` + ) + } + const wantDrift = enumeration === 'canonical' && reportIndexDrift + // includeHidden IMPLIES includeSystem (see ExportOptions.includeHidden's JSDoc) — every + // system-tier gate below reads THIS combined value, never the raw option, so + // `includeHidden` alone is always sufficient to see system-tier rows too. + const effectiveIncludeSystem = includeSystem || includeHidden + + // 1. Resolve the node-id set (+ the index's raw candidate set, only when diffing it). + // Both `enumerateAllCanonical` and `enumerateAllIndexed` receive the SAME + // `effectiveIncludeSystem`/`includeHidden` pair below, so a drift diff can never + // contain tier-policy noise — only genuine index-vs-canonical disagreement. + const { idSet, indexCandidateIds } = await resolveSelector( + reader, + storage, + selector, + effectiveIncludeSystem, + enumeration, + wantDrift, + includeHidden + ) // 2. Read canonical entities (reserved fields top-level), applying any predicate filter. + // Identical for both enumeration modes: 'canonical' only changes WHICH ids reach this + // loop, never how a candidate is verified — so the two modes can disagree on candidacy, + // never on what counts as a match. const usePredicate = hasPredicate(selector) const entityMap = new Map>() const entities: PortableGraphEntity[] = [] for (const id of idSet) { const e = await reader.get(id, { includeVectors }) if (!e) continue - if (!includeSystem && (e as any).visibility === 'system') continue + if (!effectiveIncludeSystem && (e as any).visibility === 'system') continue if (usePredicate && !matchesPredicate(e, selector)) continue entityMap.set(id, e) entities.push(toPortableGraphEntity(e, includeVectors)) } const keptIds = new Set(entityMap.keys()) - // 3. Edges per policy. - const { relations, danglingIds } = await collectEdges(reader, keptIds, edges) + // 2b. Finalize the drift report now that ground truth (keptIds) is known. + let drift: ExportIndexDrift | undefined + if (wantDrift) { + const indexIds = indexCandidateIds ?? new Set() + const canonicalOnly = [...keptIds].filter((id) => !indexIds.has(id)) + const indexOnly = [...indexIds].filter((id) => !keptIds.has(id)) + drift = { canonicalOnly, indexOnly } + if (canonicalOnly.length > 0 || indexOnly.length > 0) { + console.warn( + `[Brainy] export() index drift: ${canonicalOnly.length} canonical-only id(s) ` + + `(canon-present, the index-based enumeration missed them) and ${indexOnly.length} ` + + `index-only id(s) (index-visible, canon-absent — phantom rows). ` + + `See the returned PortableGraph's 'drift' field for the exact ids. Nothing was ` + + `auto-healed — run brain.repairIndex() to reconcile the metadata index.` + ) + } + } + + // 3. Edges per policy. Canonical mode ALSO walks verbs canonically for every + // selector (not just whole-brain) — a lost adjacency-index posting can hide a + // relation regardless of how the node set was produced. + const { relations, danglingIds } = + enumeration === 'canonical' + ? await collectEdgesCanonical(storage!, keptIds, edges, effectiveIncludeSystem, includeHidden) + : await collectEdges(reader, keptIds, edges, includeHidden) // 4. VFS blob bytes (only when requested). let blobs: Record | undefined @@ -330,6 +470,7 @@ export async function exportGraph( relations, ...(blobs && blobCount > 0 ? { blobs } : {}), ...(danglingIds && danglingIds.length > 0 ? { danglingIds } : {}), + ...(drift ? { drift } : {}), stats: { entityCount: entities.length, relationCount: relations.length, @@ -477,12 +618,36 @@ function hasPredicate(s: ExportSelector): boolean { ) } +/** + * @param reader - Generation-correct read surface. + * @param storage - Storage adapter (only touched when `enumeration:'canonical'` + * resolves the whole-brain/predicate branch). + * @param s - The export selector. + * @param includeSystem - The ALREADY-COMBINED `includeSystem || includeHidden` value + * (see `exportGraph`'s `effectiveIncludeSystem`) — whether `visibility:'system'` + * entities are wanted. + * @param enumeration - `'index'` (default) or `'canonical'` — see {@link ExportOptions.enumeration}. + * Only affects the whole-brain/predicate branch (the `else` below): structural + * selectors (`ids`/`collection`/`connected`/`vfsPath`) never rode the metadata + * index for their node set, so they resolve identically either way. + * @param wantIndexCandidates - When true (only meaningful with `enumeration:'canonical'` + * on the whole-brain/predicate branch), ALSO run the index-based walk and return + * its raw candidate set as `indexCandidateIds`, for {@link ExportIndexDrift}. + * @param includeHidden - Whether `visibility:'internal'` entities are ALSO wanted + * (see {@link ExportOptions.includeHidden}). Threaded to BOTH enumeration + * functions identically so a drift diff never contains tier-policy noise. + */ async function resolveSelector( reader: PortableGraphReader, + storage: StorageAdapter | undefined, s: ExportSelector, - includeSystem: boolean -): Promise> { + includeSystem: boolean, + enumeration: 'index' | 'canonical', + wantIndexCandidates: boolean, + includeHidden: boolean +): Promise<{ idSet: Set; indexCandidateIds?: Set }> { let idSet: Set + let indexCandidateIds: Set | undefined if (s.ids && s.ids.length) { idSet = new Set(s.ids) } else if (s.collection ?? s.memberOf) { @@ -491,20 +656,45 @@ async function resolveSelector( idSet = await resolveConnected(reader, s.connected) } else if (s.vfsPath) { idSet = await resolveVfsPath(reader, s.vfsPath, s.recursive ?? true, s.depth) + } else if (enumeration === 'canonical') { + idSet = await enumerateAllCanonical(storage!, includeSystem, includeHidden) + if (wantIndexCandidates) indexCandidateIds = await enumerateAllIndexed(reader, s, includeHidden) } else { - idSet = await enumerateAll(reader, s) + idSet = await enumerateAllIndexed(reader, s, includeHidden) } if (!includeSystem) idSet.delete(VFS_ROOT_ID) - return idSet + return { idSet, indexCandidateIds } } -/** Whole-brain / predicate enumeration via generation-correct paginated `find()`. */ -async function enumerateAll(reader: PortableGraphReader, s: ExportSelector): Promise> { +/** + * @description Whole-brain / predicate enumeration via generation-correct + * paginated `find()`. The metadata index is an acceleration structure over + * this candidate set — see {@link enumerateAllCanonical} for the storage-level + * counterpart that never consults it. + * + * @param includeHidden - When true, forwards `includeInternal: true` AND + * `includeSystem: true` into the SAME `find()` call — `find()` supports both + * flags simultaneously (confirmed via `FindParams.includeInternal`/`includeSystem` + * and `Brainy`'s `resolveHiddenIds`/`excludedVisibilityTiers`), so ONE pass + * reaches both hidden tiers; no per-tier union pass is needed. When false + * (default), neither flag is forwarded — the pre-existing behavior, preserved + * byte-identically for back-compat (`ExportOptions.includeSystem` alone never + * reached this far; see {@link ExportOptions.includeHidden}'s JSDoc). + */ +async function enumerateAllIndexed( + reader: PortableGraphReader, + s: ExportSelector, + includeHidden = false +): Promise> { const params: any = {} if (s.type !== undefined) params.type = s.type if (s.subtype !== undefined) params.subtype = s.subtype if (s.where !== undefined) params.where = s.where if (s.service !== undefined) params.service = s.service + if (includeHidden) { + params.includeInternal = true + params.includeSystem = true + } const ids = new Set() let offset = 0 // eslint-disable-next-line no-constant-condition @@ -517,6 +707,52 @@ async function enumerateAll(reader: PortableGraphReader, s: ExportSelector return ids } +/** + * @description Canonical (storage-level) counterpart of {@link enumerateAllIndexed}: + * walks every live noun directly off the storage adapter's canonical shard layout + * (`storage.getNouns()` — the same primitive `repairIndex()`'s recount and every + * index-heal walk use) instead of going through the metadata index. Guarantees + * canon-completeness — a lost or stale metadata-index posting cannot cause a + * canonical record to be silently missing from the returned set — at the cost of + * an O(N) walk regardless of selector selectivity (unlike the index path's + * O(matches)). Returns the RAW candidate id set; `exportGraph`'s caller applies + * `matchesPredicate` per-entity via `reader.get()` afterward, exactly as the index + * path does, so both paths share one predicate-evaluation code path and can only + * disagree on candidacy, never on what a match means. + * + * Mirrors `find()`'s hidden-tier policy given the SAME `includeSystem`/`includeHidden` + * pair (see {@link enumerateAllIndexed}) so the two enumeration modes produce + * identical id sets when the index is healthy, at ANY tier-visibility setting. + * + * @param includeSystem - The ALREADY-COMBINED `includeSystem || includeHidden` value. + * @param includeHidden - Whether `'internal'`-tier nouns are ALSO admitted. + */ +async function enumerateAllCanonical( + storage: StorageAdapter, + includeSystem = false, + includeHidden = false +): Promise> { + const ids = new Set() + let offset = 0 + let cursor: string | undefined + // eslint-disable-next-line no-constant-condition + while (true) { + const page = await storage.getNouns({ pagination: { limit: ENUM_PAGE, offset, cursor } }) + for (const item of page.items) { + if (item.visibility === 'internal' && !includeHidden) continue + if (item.visibility === 'system' && !includeSystem) continue + ids.add(item.id) + } + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor !== undefined) { + cursor = page.nextCursor + } else { + offset += ENUM_PAGE + } + } + return ids +} + async function resolveCollectionSubtree( reader: PortableGraphReader, rootId: string, @@ -678,19 +914,27 @@ function toPortableGraphRelation(r: Relation): PortableGraphRelation { return br } +/** + * @param includeHidden - When true, forwards `includeInternal`/`includeSystem` into + * every `related()` call so hidden-tier relations reach candidacy too — mirrors + * {@link enumerateAllIndexed}'s `includeHidden` handling, and preserves back-compat + * when false/omitted (the pre-existing, unconditional hidden-tier exclusion). + */ async function collectEdges( reader: PortableGraphReader, idSet: Set, - edges: 'induced' | 'incident' | 'none' + edges: 'induced' | 'incident' | 'none', + includeHidden = false ): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> { if (edges === 'none') return { relations: [] } + const tierOptIn = includeHidden ? { includeInternal: true, includeSystem: true } : {} const relations: PortableGraphRelation[] = [] const dangling = new Set() const seen = new Set() for (const id of idSet) { - const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT }) + const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT, ...tierOptIn }) for (const r of rels) { if (seen.has(r.id)) continue const toIn = idSet.has(r.to) @@ -703,7 +947,7 @@ async function collectEdges( if (edges === 'incident') { for (const id of idSet) { - const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT }) + const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT, ...tierOptIn }) for (const r of rels) { if (seen.has(r.id)) continue if (!idSet.has(r.from)) { @@ -718,6 +962,81 @@ async function collectEdges( return dangling.size > 0 ? { relations, danglingIds: Array.from(dangling) } : { relations } } +/** Converts a canonical verb record (as returned by `storage.getVerbs()`) into the wire shape. */ +function hnswVerbToPortableGraphRelation(v: HNSWVerbWithMetadata): PortableGraphRelation { + const br: PortableGraphRelation = { id: v.id, from: v.sourceId, to: v.targetId, type: v.verb as string } + if (v.subtype !== undefined) br.subtype = v.subtype + if (v.visibility !== undefined && v.visibility !== 'public') br.visibility = v.visibility + if (v.weight !== undefined) br.weight = v.weight + if (v.confidence !== undefined) br.confidence = v.confidence + if (v.metadata && Object.keys(v.metadata as any).length) br.metadata = v.metadata + return br +} + +/** + * @description Canonical (storage-level) counterpart of {@link collectEdges}: + * walks every live verb directly off `storage.getVerbs()` — the same primitive + * `repairIndex()`'s recount and every index-heal walk use — instead of the graph + * adjacency index (`reader.related()`), so a lost/stale adjacency posting cannot + * cause a canonical relationship to be silently dropped from the export. Used for + * EVERY selector in `enumeration:'canonical'` mode, not just the whole-brain + * branch: relations can be blinded by adjacency-index corruption regardless of + * how `idSet` (the kept node ids) was produced. + * + * Mirrors `related()`'s default hidden-tier policy (hides `'internal'` and + * `'system'` unless `includeHidden`/`includeSystem` say otherwise) so the two + * enumeration modes produce identical relation sets when the index is healthy. + * + * @param includeSystem - Whether `'system'`-tier verbs are admitted (the + * caller passes the ALREADY-combined `includeSystem || includeHidden` value — + * see `exportGraph`'s `effectiveIncludeSystem`). + * @param includeHidden - Whether `'internal'`-tier verbs are ALSO admitted. + */ +async function collectEdgesCanonical( + storage: StorageAdapter, + idSet: Set, + edges: 'induced' | 'incident' | 'none', + includeSystem = false, + includeHidden = false +): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> { + if (edges === 'none') return { relations: [] } + + const relations: PortableGraphRelation[] = [] + const dangling = new Set() + const seen = new Set() + let offset = 0 + let cursor: string | undefined + + // eslint-disable-next-line no-constant-condition + while (true) { + const page = await storage.getVerbs({ pagination: { limit: ENUM_PAGE, offset, cursor } }) + for (const v of page.items) { + if (seen.has(v.id)) continue + if (v.visibility === 'internal' && !includeHidden) continue + if (v.visibility === 'system' && !includeSystem) continue + const fromIn = idSet.has(v.sourceId) + const toIn = idSet.has(v.targetId) + if (edges === 'induced') { + if (!fromIn || !toIn) continue + } else if (!fromIn && !toIn) { + continue // 'incident': neither endpoint kept — irrelevant to this export + } + if (fromIn && !toIn) dangling.add(v.targetId) + if (toIn && !fromIn) dangling.add(v.sourceId) + seen.add(v.id) + relations.push(hnswVerbToPortableGraphRelation(v)) + } + if (!page.hasMore || page.items.length === 0) break + if (page.nextCursor !== undefined) { + cursor = page.nextCursor + } else { + offset += ENUM_PAGE + } + } + + return dangling.size > 0 ? { relations, danglingIds: Array.from(dangling) } : { relations } +} + async function collectBlobs( storage: StorageAdapter | undefined, entityMap: Map> diff --git a/src/index.ts b/src/index.ts index 70d43edc..ae8eef5c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -185,6 +185,7 @@ export type { PortableGraphRelation, ExportSelector, ExportOptions, + ExportIndexDrift, ImportOptions, ImportResult, PortableGraphValidation @@ -194,7 +195,8 @@ export { SpeculativeOverlayError, GenerationCompactedError, StoreInconsistentError, - PendingFlushDurabilityError + PendingFlushDurabilityError, + CanonicalEnumerationUnavailableError } from './db/errors.js' export type { UnreconciledRecord } from './db/errors.js' export type { diff --git a/tests/unit/db/db-portable-graph.test.ts b/tests/unit/db/db-portable-graph.test.ts index abb89553..1de9983d 100644 --- a/tests/unit/db/db-portable-graph.test.ts +++ b/tests/unit/db/db-portable-graph.test.ts @@ -9,7 +9,7 @@ * subtype-required default. */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { randomUUID } from 'node:crypto' import * as fs from 'node:fs/promises' import * as os from 'node:os' @@ -19,6 +19,7 @@ import { createTestConfig } from '../../helpers/test-factory' import { NounType, VerbType } from '../../../src/types/graphTypes' import { validatePortableGraph } from '../../../src/db/portableGraph' import type { PortableGraph } from '../../../src/db/portableGraph' +import { CanonicalEnumerationUnavailableError } from '../../../src/db/errors' describe('8.0 portable graph export/import (PortableGraph v1)', () => { let brain: Brainy @@ -293,3 +294,257 @@ describe('8.0 export includeContent (VFS blobs, filesystem)', () => { } }) }) + +describe('8.0 export enumeration:"canonical" — canon-complete against index blindness', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('(i) equals the index-based export when the index is healthy — same entity ids, relations, vectors', async () => { + const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' }) + const b = await brain.add({ data: 'Bob', type: NounType.Person, subtype: 'employee' }) + const c = await brain.add({ data: 'Acme', type: NounType.Organization, subtype: 'vendor' }) + await brain.relate({ from: a, to: b, type: VerbType.FriendOf, subtype: 'close' }) + await brain.relate({ from: a, to: c, type: VerbType.WorksWith, subtype: 'full-time' }) + + const indexExport = await brain.export({}, { includeVectors: true, enumeration: 'index' }) + const canonicalExport = await brain.export({}, { includeVectors: true, enumeration: 'canonical' }) + + expect(canonicalExport.entities.map((e) => e.id).sort()).toEqual( + indexExport.entities.map((e) => e.id).sort() + ) + expect(canonicalExport.relations.map((r) => r.id).sort()).toEqual( + indexExport.relations.map((r) => r.id).sort() + ) + expect(canonicalExport.entities.map((e) => e.id).sort()).toEqual([a, b, c].sort()) + for (const e of canonicalExport.entities) { + expect(e.vector?.length).toBeGreaterThan(0) + } + expect(canonicalExport.drift).toBeUndefined() // reportIndexDrift not requested + }) + + it('(ii) survives simulated metadata-index blindness; the index export misses the record; drift names it canonicalOnly', async () => { + const staff = await brain.add({ + data: 'Staff', + type: NounType.Person, + subtype: 'employee', + metadata: { role: 'staff' } + }) + const other = await brain.add({ + data: 'Other', + type: NounType.Person, + subtype: 'employee', + metadata: { role: 'staff' } + }) + + // Surgically poison the metadata index (the lowest-level seam the existing + // find() phantom-row guard tests use — see find-index-integrity-guard.test.ts, + // which does the mirror-image ADD case) so the predicate query + // enumeration:'index' issues (find({ type: Person })) never returns `staff` — + // a real canonical record the index has lost track of, the exact + // canon-present/index-missing state canonical mode exists to survive. + const mi = (brain as any).metadataIndex + const original = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (filter: any, opts?: any): Promise => { + const ids: string[] = await original(filter, opts) + return ids.filter((id: string) => id !== staff) + } + + try { + const indexExport = await brain.export({ type: NounType.Person }, { enumeration: 'index' }) + expect(indexExport.entities.map((e) => e.id)).not.toContain(staff) + expect(indexExport.entities.map((e) => e.id)).toContain(other) + + const canonicalExport = await brain.export( + { type: NounType.Person }, + { enumeration: 'canonical', reportIndexDrift: true } + ) + expect(canonicalExport.entities.map((e) => e.id)).toContain(staff) + expect(canonicalExport.entities.map((e) => e.id)).toContain(other) + expect(canonicalExport.drift?.canonicalOnly).toEqual([staff]) + expect(canonicalExport.drift?.indexOnly).toEqual([]) + } finally { + mi.getIdsForFilter = original + } + }) + + it('(iii) drift report shape + loud console.warn only when nonzero', async () => { + const staff = await brain.add({ data: 'Staff', type: NounType.Person, subtype: 'employee' }) + await brain.add({ data: 'Other', type: NounType.Person, subtype: 'employee' }) + + const mi = (brain as any).metadataIndex + const original = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (filter: any, opts?: any): Promise => { + const ids: string[] = await original(filter, opts) + return ids.filter((id: string) => id !== staff) + } + + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + try { + const drifted = await brain.export( + { type: NounType.Person }, + { enumeration: 'canonical', reportIndexDrift: true } + ) + expect(drifted.drift).toEqual({ canonicalOnly: [staff], indexOnly: [] }) + expect(warnSpy).toHaveBeenCalledTimes(1) + expect(warnSpy.mock.calls[0].join(' ')).toMatch(/drift/i) + } finally { + mi.getIdsForFilter = original + warnSpy.mockClear() + } + + // Healthy index: drift is reported (both lists present) but never warned about. + try { + const healthy = await brain.export( + { type: NounType.Person }, + { enumeration: 'canonical', reportIndexDrift: true } + ) + expect(healthy.drift).toEqual({ canonicalOnly: [], indexOnly: [] }) + expect(warnSpy).not.toHaveBeenCalled() + } finally { + warnSpy.mockRestore() + } + }) + + it('(iv) throws CanonicalEnumerationUnavailableError on a historical asOf() view and a speculative with() overlay', async () => { + const a = '22222222-2222-4222-8222-222222222222' + const b = '33333333-3333-4333-8333-333333333333' + await brain.transact([{ op: 'add', id: a, data: 'First', type: NounType.Thing, subtype: 'x' }]) + const g1 = brain.generation() + await brain.transact([{ op: 'add', id: b, data: 'Second', type: NounType.Thing, subtype: 'x' }]) + + const past = await brain.asOf(g1) + try { + await expect(past.export({}, { enumeration: 'canonical' })).rejects.toThrow( + CanonicalEnumerationUnavailableError + ) + // The default (index) mode is unaffected — still a valid time-travel export. + const backup = await past.export() + expect(backup.entities.map((e) => e.id)).toContain(a) + } finally { + await past.release() + } + + const speculativeId = '11111111-1111-4111-8111-111111111111' + const view = await brain.now().with([ + { op: 'add', id: speculativeId, data: 'Speculative', type: NounType.Thing, subtype: 'x' } + ]) + try { + await expect(view.export({}, { enumeration: 'canonical' })).rejects.toThrow( + CanonicalEnumerationUnavailableError + ) + } finally { + await view.release() + } + }) + + it('throws a plain Error when enumeration:"canonical" has no storage adapter to walk', async () => { + const { exportGraph } = await import('../../../src/db/portableGraph') + const readerOnly = { get: async () => null, find: async () => [], related: async () => [] } + await expect( + exportGraph(readerOnly as any, undefined, {}, { enumeration: 'canonical' }) + ).rejects.toThrow(/enumeration:'canonical' requires a storage adapter/) + }) +}) + +describe('8.0 export includeHidden — every visibility tier for migration-grade canon completeness', () => { + // The fixed-id VFS root Brainy.init() always creates is the one 'system'-visibility + // entity a consumer can rely on existing (visibility:'system' is not settable via the + // public add() API — "intentionally not accepted", per AddParams.visibility's doc). + const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('canonical + includeHidden carries an internal row AND the system row; round-trips through import', async () => { + const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' }) + const internalId = await brain.add({ + data: 'Internal', + type: NounType.Thing, + subtype: 'x', + visibility: 'internal' + }) + + const migrationExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true }) + const ids = migrationExport.entities.map((e) => e.id) + expect(ids).toContain(publicId) + expect(ids).toContain(internalId) + expect(ids).toContain(VFS_ROOT_ID) + expect(migrationExport.entities.find((e) => e.id === internalId)?.visibility).toBe('internal') + expect(migrationExport.entities.find((e) => e.id === VFS_ROOT_ID)?.visibility).toBe('system') + + const target = new Brainy(createTestConfig()) + await target.init() + try { + const result = await target.import(migrationExport) + expect(result.errors).toHaveLength(0) + expect((await target.get(internalId))?.visibility).toBe('internal') + } finally { + await target.close() + } + }) + + it('default export (includeHidden omitted) still excludes both hidden tiers — pins today\'s behavior', async () => { + const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' }) + const internalId = await brain.add({ + data: 'Internal', + type: NounType.Thing, + subtype: 'x', + visibility: 'internal' + }) + + for (const opts of [{ enumeration: 'index' as const }, { enumeration: 'canonical' as const }]) { + const backup = await brain.export({}, opts) + const ids = backup.entities.map((e) => e.id) + expect(ids).toContain(publicId) + expect(ids).not.toContain(internalId) + expect(ids).not.toContain(VFS_ROOT_ID) + } + }) + + it('index mode + includeHidden also reaches both tiers — find() takes includeInternal + includeSystem in one pass', async () => { + const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' }) + const internalId = await brain.add({ + data: 'Internal', + type: NounType.Thing, + subtype: 'x', + visibility: 'internal' + }) + + const indexExport = await brain.export({}, { enumeration: 'index', includeHidden: true }) + const canonicalExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true }) + + const indexIds = indexExport.entities.map((e) => e.id).sort() + const canonicalIds = canonicalExport.entities.map((e) => e.id).sort() + expect(indexIds).toEqual(canonicalIds) + expect(indexIds).toContain(publicId) + expect(indexIds).toContain(internalId) + expect(indexIds).toContain(VFS_ROOT_ID) + }) + + it('drift stays pure under includeHidden — no tier-policy noise when the index is healthy', async () => { + await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' }) + await brain.add({ data: 'Internal', type: NounType.Thing, subtype: 'x', visibility: 'internal' }) + + const audited = await brain.export( + {}, + { enumeration: 'canonical', includeHidden: true, reportIndexDrift: true } + ) + expect(audited.drift).toEqual({ canonicalOnly: [], indexOnly: [] }) + }) +})