From fc7f110479116499fa5c0e463facc8a51330596a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 30 Jun 2026 09:53:10 -0700 Subject: [PATCH] feat(8.0): version-handshake marker (formatInfo + indexEpoch) for whole-brain auto-upgrade Add _system/brain-format.json { dataFormat, indexEpoch } + a sync brain.formatInfo() accessor + a compiled EXPECTED_INDEX_EPOCH, the surface a native provider reads at init() to drive whole-brain auto-upgrade, and brainy's own derived-index rebuild-on-format-drift trigger (closing the gap where JS indexes rebuilt only on size()===0, never on a format-version change). indexEpoch is shared and lockstep-bumped with the native provider on any coordinated release whose on-disk derived-index format changes; dataFormat is brainy-owned. On open, the marker is read in the store-open phase BEFORE provider construction (so formatInfo() is synchronously available at the provider's init); a drifted or absent epoch rebuilds the derived indexes from canonical records, and the marker is stamped only AFTER the rebuild verifies (build-new -> verify -> stamp; a crash before the stamp re-triggers the idempotent rebuild). brainFormat.ts is the single source of the shared constants. 6-case test; 1724 unit green. --- src/brainy.ts | 128 ++++++++- src/storage/brainFormat.ts | 136 ++++++++++ .../brainy/brain-format-handshake.test.ts | 243 ++++++++++++++++++ 3 files changed, 499 insertions(+), 8 deletions(-) create mode 100644 src/storage/brainFormat.ts create mode 100644 tests/unit/brainy/brain-format-handshake.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index e8a38607..25a7669a 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -17,6 +17,13 @@ import type { StorageOptions } from './storage/storageFactory.js' import { rebuildCounts } from './utils/rebuildCounts.js' import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js' import { BaseStorage } from './storage/baseStorage.js' +import { + CURRENT_DATA_FORMAT, + EXPECTED_INDEX_EPOCH, + readBrainFormat, + writeBrainFormat +} from './storage/brainFormat.js' +import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { @@ -407,6 +414,24 @@ export class Brainy implements BrainyInterface { private _graphAdjacencyVerified = false /** Re-entrancy guard: a verify (rebuild → reads) is in flight. */ private _graphAdjacencyVerifying = false + /** + * 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat} + * marker (`_system/brain-format.json`) as read during the store-open phase, + * or `null` for a pre-handshake / brand-new brain. Populated BEFORE any + * derived index or native provider is constructed, so {@link formatInfo} + * answers synchronously when the provider reads it at its own init(). + * Refreshed to the current marker once it is (re)stamped. + */ + private _brainFormat: BrainFormat | null = null + /** + * True when the on-disk {@link _brainFormat} is absent OR carries an + * `indexEpoch` different from {@link EXPECTED_INDEX_EPOCH} — i.e. the derived + * JS indexes on disk predate this build and must be rebuilt from the + * canonical records (the epoch-drift rebuild trigger that closes the gap + * where JS indexes rebuilt only on `size()===0`, never on a format change). + * Cleared once the rebuild verifies and the marker is re-stamped. + */ + private _indexEpochStale = false // Sub-APIs (lazy-loaded) private _nlp?: NaturalLanguageProcessor @@ -799,6 +824,20 @@ export class Brainy implements BrainyInterface { readOnly: this.config.mode === 'reader' }) + // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format + // marker (`_system/brain-format.json`) into an in-memory field NOW — + // after the store-open phase, but BEFORE any derived index or native + // provider is constructed below — so `formatInfo()` is populated when the + // provider reads it synchronously at its own init(). A drifted or absent + // `indexEpoch` means the on-disk derived-index format predates this build + // (the derived JS indexes are stale); `rebuildIndexesIfNeeded()` rebuilds + // them from the canonical records and then re-stamps the marker AFTER the + // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old / + // absent marker, so the next open idempotently re-rebuilds). + this._brainFormat = await readBrainFormat(this.storage) + this._indexEpochStale = + this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH + // Provider: embeddings (reassign embedder if plugin provides one) const embeddingProvider = this.pluginRegistry.getProvider('embeddings') if (embeddingProvider) { @@ -6339,6 +6378,35 @@ export class Brainy implements BrainyInterface { return this.generationStore.generation() } + /** + * @description The data-layer + derived-index format this brain instance runs + * as — the SYNCHRONOUS half of the 7.x → 8.0 version handshake. Returns the + * compiled {@link CURRENT_DATA_FORMAT} / {@link EXPECTED_INDEX_EPOCH} + * constants (the single source of truth shared with the native provider, in + * `src/storage/brainFormat.ts`): after `init()` the brain has reconciled any + * on-disk drift and IS running the current format, so this reports what it + * runs as, not necessarily the (possibly older) marker it opened. + * + * This is a pure in-memory constant read — no `await`, no storage I/O — by + * design: a native metadata/index provider calls it synchronously at its own + * `init()` (which runs during this brain's provider-construction phase, after + * the marker has been loaded) to confirm "running data-format X, index epoch + * N" before binding its native readers. + * + * The on-disk marker (`_system/brain-format.json`) is reconciled separately: + * on open Brainy compares the on-disk `indexEpoch` against + * {@link EXPECTED_INDEX_EPOCH}; a mismatch or an absent marker rebuilds the + * derived indexes and re-stamps the marker (see `rebuildIndexesIfNeeded`). + * + * @returns `{ dataFormat, indexEpoch }` for the running build. + * @example + * const { dataFormat, indexEpoch } = brain.formatInfo() + * // dataFormat === '8.0', indexEpoch === 1 + */ + formatInfo(): BrainFormat { + return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + } + /** * @description Read the reified transaction log — one entry per committed * generation, carrying the committed generation, the commit timestamp, and @@ -12946,8 +13014,12 @@ export class Brainy implements BrainyInterface { } // OPTIMIZATION: Instant check - if index already has data, skip immediately - // This gives 0s startup for warm restarts (vs 50-100ms of async checks) - if (this.index.size() > 0 && !force) { + // This gives 0s startup for warm restarts (vs 50-100ms of async checks). + // The epoch-drift guard suppresses this fast path: an index that loaded + // eagerly (size()>0) but whose on-disk format predates this build + // (`_indexEpochStale`) must still rebuild — that is the exact gap this + // handshake closes (rebuild on a format change, not only on size()===0). + if (this.index.size() > 0 && !force && !this._indexEpochStale) { if (!this.config.silent) { console.log( `✅ Index already populated (${this.index.size().toLocaleString()} entities) - 0s startup!` @@ -12966,6 +13038,10 @@ export class Brainy implements BrainyInterface { if (force && !this.config.silent) { console.log('✅ Storage empty - no rebuild needed') } + // A fresh / empty brain's (empty) derived indexes are trivially current + // for this build's epoch — stamp the version marker so the next open + // recognises it as current and skips the drift rebuild. + await this.stampBrainFormatIfNeeded() return } @@ -12988,10 +13064,16 @@ export class Brainy implements BrainyInterface { graphIndexSize === 0 || (typeof graphIsReady.isReady === 'function' && !graphIsReady.isReady()) + // Epoch-drift trigger: a format-version change makes EVERY derived index + // suspect even when each is non-empty, so it forces a rebuild of all + // three from the canonical records (not just the empty ones below). + const epochStale = this._indexEpochStale + const needsRebuild = metadataStats.totalEntries === 0 || hnswIndexSize === 0 || - needsGraphRebuild + needsGraphRebuild || + epochStale if (!needsRebuild && !force) { // All indexes already populated, no rebuild needed @@ -13029,17 +13111,20 @@ export class Brainy implements BrainyInterface { // graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints // and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s // ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()). - if (needsGraphRebuild) { + if (needsGraphRebuild || epochStale) { await this.hydrateIdMapperForGraphRebuild() } // Rebuild all 3 indexes in parallel for performance - // Indexes load their data from storage (no recomputation) + // Indexes load their data from storage (no recomputation). An epoch drift + // (`epochStale`) rebuilds ALL three regardless of their current size — + // the on-disk format changed, so every derived index is rebuilt from the + // canonical records. const rebuildStartTime = Date.now() await Promise.all([ - metadataStats.totalEntries === 0 ? this.metadataIndex.rebuild() : Promise.resolve(), - hnswIndexSize === 0 ? this.index.rebuild() : Promise.resolve(), - needsGraphRebuild ? this.graphIndex.rebuild() : Promise.resolve() + metadataStats.totalEntries === 0 || epochStale ? this.metadataIndex.rebuild() : Promise.resolve(), + hnswIndexSize === 0 || epochStale ? this.index.rebuild() : Promise.resolve(), + needsGraphRebuild || epochStale ? this.graphIndex.rebuild() : Promise.resolve() ]) const rebuildDuration = Date.now() - rebuildStartTime @@ -13066,6 +13151,14 @@ export class Brainy implements BrainyInterface { console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`) } + // 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes + // have now rebuilt and verified, so they match this build's epoch — + // re-stamp the marker LAST, only here. A crash anywhere above leaves the + // old / absent marker, so the next open re-detects the drift and re-runs + // the (idempotent) rebuild; the marker is never advanced ahead of the + // indexes it certifies. A no-op when the epoch was already current. + await this.stampBrainFormatIfNeeded() + } catch (error) { // A storage READ failure here is surfaced by getNouns/getVerbs as a named // BrainyError — it means we could not even read the store to decide whether @@ -13083,6 +13176,25 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Re-stamp the 8.0 ⇄ native-provider version-handshake marker + * (`_system/brain-format.json`) to this build's {@link CURRENT_DATA_FORMAT} / + * {@link EXPECTED_INDEX_EPOCH} — but ONLY when the on-disk marker was absent + * or carried a drifted epoch ({@link _indexEpochStale}). Called at the + * verified-completion points of {@link rebuildIndexesIfNeeded} (after the + * derived-index rebuild, or on the empty-storage fast path), so the marker is + * never advanced ahead of the indexes it certifies — a crash before this + * point re-triggers the idempotent rebuild on the next open. Readers never + * write, so this is a no-op in reader mode. + */ + private async stampBrainFormatIfNeeded(): Promise { + if (this.isReadOnly) return + if (!this._indexEpochStale) return + await writeBrainFormat(this.storage) + this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + this._indexEpochStale = false + } + /** * Check health of metadata indexes * diff --git a/src/storage/brainFormat.ts b/src/storage/brainFormat.ts new file mode 100644 index 00000000..2e6488e9 --- /dev/null +++ b/src/storage/brainFormat.ts @@ -0,0 +1,136 @@ +/** + * @module storage/brainFormat + * @description The 7.x → 8.0 version-handshake marker: the small persisted + * artifact at `_system/brain-format.json` that lets Brainy and a native + * metadata/index provider agree, at open time, on the on-disk format of the + * brain AND of its derived indexes. + * + * Two fields, two owners: + * + * - `dataFormat` — **Brainy-owned**. The data-layer version string (the + * canonical entity / relationship / generation-record layout). Brainy bumps + * it when that data layout changes; a native provider reads it only to + * confirm the major line it is binding against (e.g. "this is an 8.0 brain"). + * - `indexEpoch` — **SHARED**. A monotonic integer that Brainy and the native + * provider bump TOGETHER, in lockstep, on any coordinated release where the + * on-disk format of ANY derived index (the HNSW vectors, the metadata + * postings, or the graph adjacency) changes. It is the single switch that + * declares "every derived index written by an older build is stale and must + * be rebuilt from the canonical records." + * + * Open-time handshake. On open, Brainy compares the on-disk `indexEpoch` + * against the compiled {@link EXPECTED_INDEX_EPOCH}. A mismatch — OR an absent + * marker (a pre-handshake brain, or a brand-new store) — means the derived + * indexes on disk predate this build, so Brainy rebuilds them from the + * canonical records and only THEN re-stamps the marker. The provider reads the + * same surface synchronously through `brain.formatInfo()` at its own provider + * init() to confirm "running data-format X, index epoch N" before binding its + * native readers. + * + * Lockstep contract. NEVER bump {@link EXPECTED_INDEX_EPOCH} unilaterally. It + * advances only on a coordinated release whose on-disk derived-index format + * actually changed, and both projects ship the SAME new value in the same + * release — so a brain written by either side is recognised as current by the + * other, and a brain written by an older build of either side is recognised as + * stale and rebuilt. The constant living here makes this module the single + * source of truth both sides reference. + * + * Non-destructive stamping. The marker is the LAST thing written, AFTER the + * rebuild has verified. A crash between the rebuild and the stamp leaves the + * old / absent marker on disk, so the next open re-detects the drift and + * re-runs the (idempotent) rebuild — the marker is never advanced ahead of the + * indexes it certifies. This mirrors the generational record layer's + * "build-new → verify → atomic-rename" commit discipline + * (`src/db/generationStore.ts`). + */ + +/** + * @description The narrow storage surface the marker helpers need — the + * raw-object read/write primitives every `BaseStorage` adapter implements (the + * same surface the generational record layer uses for `_system/` artifacts). + * Declared structurally so this module carries no runtime dependency on the + * adapter class. + */ +export interface BrainFormatStorage { + /** Read a raw object at a storage-root-relative path (`null` if absent). */ + readRawObject(path: string): Promise + /** Write a raw object at a storage-root-relative path (atomic tmp+rename on disk). */ + writeRawObject(path: string, data: unknown): Promise +} + +/** Storage-root-relative path of the version-handshake marker. */ +export const BRAIN_FORMAT_PATH = '_system/brain-format.json' + +/** + * @description The compiled derived-index format epoch this build expects on + * disk. SHARED and lockstep-bumped with the native provider: advance it (in + * BOTH projects, to the same value, in a coordinated release) on any change to + * the on-disk format of any derived index — never on one side alone. Start = 1 + * (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an + * absent marker — triggers a full derived-index rebuild on open. + */ +export const EXPECTED_INDEX_EPOCH = 1 + +/** + * @description The data-layer format string this build writes and runs as. + * Brainy-owned; bumped when the canonical data layout changes. Start = '8.0'. + */ +export const CURRENT_DATA_FORMAT = '8.0' + +/** + * @description The persisted shape of `_system/brain-format.json` — the + * value `brain.formatInfo()` returns for the running brain, and the value + * read back from disk to drive the epoch-drift rebuild trigger. + */ +export interface BrainFormat { + /** Brainy-owned data-layer version string (e.g. `'8.0'`). */ + dataFormat: string + /** Shared, lockstep-bumped derived-index format epoch (e.g. `1`). */ + indexEpoch: number +} + +/** + * @description Read the on-disk version-handshake marker, or `null` when it is + * absent (a pre-handshake brain, or a brand-new store). A malformed marker + * (not an object, missing either field, or a non-finite epoch) is also treated + * as `null` — a corrupt marker forces a safe rebuild rather than trusting a bad + * epoch. + * @param storage - The brain's storage adapter (raw-object surface). + * @returns The parsed {@link BrainFormat}, or `null`. + * @example + * const onDisk = await readBrainFormat(brain.storage) + * const stale = onDisk === null || onDisk.indexEpoch !== EXPECTED_INDEX_EPOCH + */ +export async function readBrainFormat( + storage: Pick +): Promise { + const raw = (await storage.readRawObject(BRAIN_FORMAT_PATH)) as Partial | null + if (raw === null || typeof raw !== 'object') return null + if ( + typeof raw.dataFormat !== 'string' || + typeof raw.indexEpoch !== 'number' || + !Number.isFinite(raw.indexEpoch) + ) { + return null + } + return { dataFormat: raw.dataFormat, indexEpoch: raw.indexEpoch } +} + +/** + * @description Stamp this build's {@link CURRENT_DATA_FORMAT} / + * {@link EXPECTED_INDEX_EPOCH} to `_system/brain-format.json` (an atomic + * tmp+rename on the filesystem adapter). MUST be called only AFTER the + * derived-index rebuild has verified: the marker certifies the indexes on + * disk, so advancing it ahead of them would suppress the rebuild a future open + * needs (the non-destructive contract — see the module docs). + * @param storage - The brain's storage adapter (raw-object surface). + */ +export async function writeBrainFormat( + storage: Pick +): Promise { + const marker: BrainFormat = { + dataFormat: CURRENT_DATA_FORMAT, + indexEpoch: EXPECTED_INDEX_EPOCH + } + await storage.writeRawObject(BRAIN_FORMAT_PATH, marker) +} diff --git a/tests/unit/brainy/brain-format-handshake.test.ts b/tests/unit/brainy/brain-format-handshake.test.ts new file mode 100644 index 00000000..e5fce73d --- /dev/null +++ b/tests/unit/brainy/brain-format-handshake.test.ts @@ -0,0 +1,243 @@ +/** + * @module brain-format-handshake.test + * @description Tests for the 7.x → 8.0 version-handshake marker (GA #30): the + * `_system/brain-format.json` surface cor reads to drive whole-brain + * auto-upgrade, and Brainy's own JS-index rebuild-on-format-drift trigger. + * + * The contract (converged with the cor team — see `src/storage/brainFormat.ts`): + * + * - `formatInfo()` is a SYNC accessor returning the running build's + * `{ dataFormat, indexEpoch }` (the compiled constants). + * - On open, an on-disk `indexEpoch` that differs from `EXPECTED_INDEX_EPOCH`, + * or an absent marker, means the derived JS indexes are stale and must be + * rebuilt from the canonical records, then the marker is re-stamped — but + * only AFTER the rebuild verifies (non-destructive: a crash mid-rebuild + * leaves the old / absent marker, so the next open idempotently re-rebuilds). + * + * Persistence note: the derived JS indexes are in-memory and cold-load via + * `rebuild()` on every reopen, so "did a rebuild happen" is not, on its own, a + * signal that the EPOCH triggered it. The epoch-force (rebuild past the warm + * `size()>0` fast path) is therefore proven by a focused white-box test; the + * reopen tests assert the observable handshake effects — the marker content, + * whether it was re-stamped, and the rebuild-before-stamp ordering. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { BaseStorage } from '../../../src/storage/baseStorage.js' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js' +import { + BRAIN_FORMAT_PATH, + CURRENT_DATA_FORMAT, + EXPECTED_INDEX_EPOCH +} from '../../../src/storage/brainFormat.js' + +const CURRENT_MARKER = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + +const tempDirs: string[] = [] +const brains: Brainy[] = [] + +function makeTempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-format-handshake-')) + tempDirs.push(dir) + return dir +} + +/** Open (and track) a filesystem brain rooted at `dir`. */ +async function openFsBrain(dir: string): Promise { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + brains.push(brain) + return brain +} + +/** Read the on-disk marker through a fresh raw adapter (after the brain closed). */ +async function readMarkerFromDisk(dir: string): Promise { + const adapter = new FileSystemStorage(dir) + await adapter.init() + return adapter.readRawObject(BRAIN_FORMAT_PATH) +} + +/** Overwrite (or delete, when `marker` is null) the on-disk marker between reopens. */ +async function seedMarkerOnDisk(dir: string, marker: unknown | null): Promise { + const adapter = new FileSystemStorage(dir) + await adapter.init() + if (marker === null) { + await adapter.deleteRawObject(BRAIN_FORMAT_PATH) + } else { + await adapter.writeRawObject(BRAIN_FORMAT_PATH, marker) + } +} + +afterEach(async () => { + for (const brain of brains.splice(0)) { + try { + await brain.close() + } catch { + // already closed by the test + } + } + vi.restoreAllMocks() + for (const dir of tempDirs.splice(0)) { + await fs.promises.rm(dir, { recursive: true, force: true }) + } +}) + +describe('8.0 ⇄ cor version-handshake marker (_system/brain-format.json)', () => { + // (a) Fresh brain → formatInfo() is the current format; the marker is written. + it('(a) fresh brain: formatInfo() reports the current format and writes the marker', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + brains.push(brain) + + expect(brain.formatInfo()).toEqual(CURRENT_MARKER) + + const onDisk = await (brain as unknown as { + storage: { readRawObject(p: string): Promise } + }).storage.readRawObject(BRAIN_FORMAT_PATH) + expect(onDisk).toEqual(CURRENT_MARKER) + }) + + // (b) Reopen a current-epoch brain → no re-stamp; formatInfo() consistent. + it('(b) reopen at the current epoch: no re-stamp, formatInfo() stays consistent', async () => { + const dir = makeTempDir() + const first = await openFsBrain(dir) + await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } }) + await first.close() + brains.splice(brains.indexOf(first), 1) + + // The on-disk marker is present and current; reopening must NOT rewrite it. + const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject') + const second = await openFsBrain(dir) + + const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH) + expect(stampWrites.length).toBe(0) + expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false) + expect(second.formatInfo()).toEqual(CURRENT_MARKER) + }) + + // (c) Stale on-disk epoch → derived-index rebuild, THEN re-stamp (rebuild before stamp). + it('(c) stale on-disk epoch: rebuilds the derived indexes, then re-stamps (rebuild BEFORE stamp)', async () => { + const dir = makeTempDir() + const first = await openFsBrain(dir) + await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } }) + await first.close() + brains.splice(brains.indexOf(first), 1) + + // Simulate a brain written by an OLDER build: drift the epoch back to 0. + await seedMarkerOnDisk(dir, { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH - 1 }) + + // Spy AFTER the seed write so only the reopen's calls are observed. Both + // spies call through; vitest's `invocationCallOrder` is a single global + // counter, so cross-spy ordering is comparable. + const rebuildSpy = vi.spyOn(MetadataIndexManager.prototype, 'rebuild') + const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject') + + const second = await openFsBrain(dir) + + // The marker was re-stamped exactly once, to the EXPECTED epoch. + const stampOrders = writeSpy.mock.calls + .map((call, i) => ({ path: call[0] as string, order: writeSpy.mock.invocationCallOrder[i] })) + .filter((c) => c.path === BRAIN_FORMAT_PATH) + expect(stampOrders.length).toBe(1) + expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false) + + // The derived index rebuilt, and the rebuild happened BEFORE the stamp + // (non-destructive ordering: the marker only advances once the rebuild that + // it certifies has run). + expect(rebuildSpy).toHaveBeenCalled() + const firstRebuildOrder = rebuildSpy.mock.invocationCallOrder[0] + expect(firstRebuildOrder).toBeLessThan(stampOrders[0].order) + + await second.close() + brains.splice(brains.indexOf(second), 1) + + // And the new marker is durable on disk at the EXPECTED epoch. + expect(await readMarkerFromDisk(dir)).toEqual(CURRENT_MARKER) + }) + + // (d) No marker (pre-handshake brain) → treated as stale → rebuild + stamp. + it('(d) no marker (pre-handshake brain): treated as stale → rebuilds and stamps the marker', async () => { + const dir = makeTempDir() + const first = await openFsBrain(dir) + await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } }) + await first.close() + brains.splice(brains.indexOf(first), 1) + + // A brain written before the handshake existed has NO marker on disk. + await seedMarkerOnDisk(dir, null) + expect(await readMarkerFromDisk(dir)).toBeNull() + + const rebuildSpy = vi.spyOn(MetadataIndexManager.prototype, 'rebuild') + const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject') + + const second = await openFsBrain(dir) + + expect(rebuildSpy).toHaveBeenCalled() + const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH) + expect(stampWrites.length).toBe(1) + expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false) + + await second.close() + brains.splice(brains.indexOf(second), 1) + expect(await readMarkerFromDisk(dir)).toEqual(CURRENT_MARKER) + }) + + // The epoch-drift trigger itself: a format change forces a rebuild past the + // warm-index fast path (the gap this feature closes); a current epoch does not. + it('epoch drift forces a rebuild of all three indexes past the warm fast path; current epoch does not', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + brains.push(brain) + + // Warm the indexes (size() > 0) so the fast-path early return is live. + await brain.add({ data: 'a', type: NounType.Document, metadata: { k: 1 } }) + await brain.add({ data: 'b', type: NounType.Document, metadata: { k: 2 } }) + const internals = brain as unknown as { + index: { size(): number; rebuild(...a: unknown[]): Promise } + metadataIndex: { rebuild(...a: unknown[]): Promise } + graphIndex: { rebuild(...a: unknown[]): Promise } + _indexEpochStale: boolean + rebuildIndexesIfNeeded(force?: boolean): Promise + } + expect(internals.index.size()).toBeGreaterThan(0) + + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + // Current epoch: the warm fast path skips every rebuild. + internals._indexEpochStale = false + await internals.rebuildIndexesIfNeeded() + expect(idxSpy).not.toHaveBeenCalled() + expect(miSpy).not.toHaveBeenCalled() + expect(giSpy).not.toHaveBeenCalled() + + // Drifted epoch: all three rebuild even though the indexes are non-empty. + internals._indexEpochStale = true + await internals.rebuildIndexesIfNeeded() + expect(idxSpy).toHaveBeenCalledTimes(1) + expect(miSpy).toHaveBeenCalledTimes(1) + expect(giSpy).toHaveBeenCalledTimes(1) + // The stamp cleared the stale flag after the verified rebuild. + expect(internals._indexEpochStale).toBe(false) + }) + + // (e) formatInfo() is synchronously available immediately after init (no await). + it('(e) formatInfo() is synchronously available immediately after init', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + brains.push(brain) + + // No await on the call itself — it is a pure in-memory constant read. + const info = brain.formatInfo() + expect(info).toEqual(CURRENT_MARKER) + expect(typeof info.indexEpoch).toBe('number') + expect(typeof info.dataFormat).toBe('string') + }) +})