feat(8.0): version-handshake marker (formatInfo + indexEpoch) for whole-brain auto-upgrade

Add _system/brain-format.json { dataFormat, indexEpoch } + a sync brain.formatInfo()
accessor + a compiled EXPECTED_INDEX_EPOCH, the surface a native provider reads at
init() to drive whole-brain auto-upgrade, and brainy's own derived-index
rebuild-on-format-drift trigger (closing the gap where JS indexes rebuilt only on
size()===0, never on a format-version change).

indexEpoch is shared and lockstep-bumped with the native provider on any coordinated
release whose on-disk derived-index format changes; dataFormat is brainy-owned. On open,
the marker is read in the store-open phase BEFORE provider construction (so formatInfo()
is synchronously available at the provider's init); a drifted or absent epoch rebuilds
the derived indexes from canonical records, and the marker is stamped only AFTER the
rebuild verifies (build-new -> verify -> stamp; a crash before the stamp re-triggers the
idempotent rebuild). brainFormat.ts is the single source of the shared constants.
6-case test; 1724 unit green.

tests/unit/brainy/brain-format-handshake.test.ts

@@ -0,0 +1,243 @@
+/**
+ * @module brain-format-handshake.test
+ * @description Tests for the 7.x → 8.0 version-handshake marker (GA #30): the
+ * `_system/brain-format.json` surface cor reads to drive whole-brain
+ * auto-upgrade, and Brainy's own JS-index rebuild-on-format-drift trigger.
+ *
+ * The contract (converged with the cor team — see `src/storage/brainFormat.ts`):
+ *
+ * - `formatInfo()` is a SYNC accessor returning the running build's
+ *   `{ dataFormat, indexEpoch }` (the compiled constants).
+ * - On open, an on-disk `indexEpoch` that differs from `EXPECTED_INDEX_EPOCH`,
+ *   or an absent marker, means the derived JS indexes are stale and must be
+ *   rebuilt from the canonical records, then the marker is re-stamped — but
+ *   only AFTER the rebuild verifies (non-destructive: a crash mid-rebuild
+ *   leaves the old / absent marker, so the next open idempotently re-rebuilds).
+ *
+ * Persistence note: the derived JS indexes are in-memory and cold-load via
+ * `rebuild()` on every reopen, so "did a rebuild happen" is not, on its own, a
+ * signal that the EPOCH triggered it. The epoch-force (rebuild past the warm
+ * `size()>0` fast path) is therefore proven by a focused white-box test; the
+ * reopen tests assert the observable handshake effects — the marker content,
+ * whether it was re-stamped, and the rebuild-before-stamp ordering.
+ */
+
+import { describe, it, expect, afterEach, vi } from 'vitest'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { Brainy } from '../../../src/index.js'
+import { NounType } from '../../../src/types/graphTypes.js'
+import { BaseStorage } from '../../../src/storage/baseStorage.js'
+import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
+import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js'
+import {
+  BRAIN_FORMAT_PATH,
+  CURRENT_DATA_FORMAT,
+  EXPECTED_INDEX_EPOCH
+} from '../../../src/storage/brainFormat.js'
+
+const CURRENT_MARKER = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
+
+const tempDirs: string[] = []
+const brains: Brainy[] = []
+
+function makeTempDir(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-format-handshake-'))
+  tempDirs.push(dir)
+  return dir
+}
+
+/** Open (and track) a filesystem brain rooted at `dir`. */
+async function openFsBrain(dir: string): Promise<Brainy> {
+  const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
+  await brain.init()
+  brains.push(brain)
+  return brain
+}
+
+/** Read the on-disk marker through a fresh raw adapter (after the brain closed). */
+async function readMarkerFromDisk(dir: string): Promise<unknown> {
+  const adapter = new FileSystemStorage(dir)
+  await adapter.init()
+  return adapter.readRawObject(BRAIN_FORMAT_PATH)
+}
+
+/** Overwrite (or delete, when `marker` is null) the on-disk marker between reopens. */
+async function seedMarkerOnDisk(dir: string, marker: unknown | null): Promise<void> {
+  const adapter = new FileSystemStorage(dir)
+  await adapter.init()
+  if (marker === null) {
+    await adapter.deleteRawObject(BRAIN_FORMAT_PATH)
+  } else {
+    await adapter.writeRawObject(BRAIN_FORMAT_PATH, marker)
+  }
+}
+
+afterEach(async () => {
+  for (const brain of brains.splice(0)) {
+    try {
+      await brain.close()
+    } catch {
+      // already closed by the test
+    }
+  }
+  vi.restoreAllMocks()
+  for (const dir of tempDirs.splice(0)) {
+    await fs.promises.rm(dir, { recursive: true, force: true })
+  }
+})
+
+describe('8.0 ⇄ cor version-handshake marker (_system/brain-format.json)', () => {
+  // (a) Fresh brain → formatInfo() is the current format; the marker is written.
+  it('(a) fresh brain: formatInfo() reports the current format and writes the marker', async () => {
+    const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    brains.push(brain)
+
+    expect(brain.formatInfo()).toEqual(CURRENT_MARKER)
+
+    const onDisk = await (brain as unknown as {
+      storage: { readRawObject(p: string): Promise<unknown> }
+    }).storage.readRawObject(BRAIN_FORMAT_PATH)
+    expect(onDisk).toEqual(CURRENT_MARKER)
+  })
+
+  // (b) Reopen a current-epoch brain → no re-stamp; formatInfo() consistent.
+  it('(b) reopen at the current epoch: no re-stamp, formatInfo() stays consistent', async () => {
+    const dir = makeTempDir()
+    const first = await openFsBrain(dir)
+    await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } })
+    await first.close()
+    brains.splice(brains.indexOf(first), 1)
+
+    // The on-disk marker is present and current; reopening must NOT rewrite it.
+    const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject')
+    const second = await openFsBrain(dir)
+
+    const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH)
+    expect(stampWrites.length).toBe(0)
+    expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false)
+    expect(second.formatInfo()).toEqual(CURRENT_MARKER)
+  })
+
+  // (c) Stale on-disk epoch → derived-index rebuild, THEN re-stamp (rebuild before stamp).
+  it('(c) stale on-disk epoch: rebuilds the derived indexes, then re-stamps (rebuild BEFORE stamp)', async () => {
+    const dir = makeTempDir()
+    const first = await openFsBrain(dir)
+    await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } })
+    await first.close()
+    brains.splice(brains.indexOf(first), 1)
+
+    // Simulate a brain written by an OLDER build: drift the epoch back to 0.
+    await seedMarkerOnDisk(dir, { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH - 1 })
+
+    // Spy AFTER the seed write so only the reopen's calls are observed. Both
+    // spies call through; vitest's `invocationCallOrder` is a single global
+    // counter, so cross-spy ordering is comparable.
+    const rebuildSpy = vi.spyOn(MetadataIndexManager.prototype, 'rebuild')
+    const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject')
+
+    const second = await openFsBrain(dir)
+
+    // The marker was re-stamped exactly once, to the EXPECTED epoch.
+    const stampOrders = writeSpy.mock.calls
+      .map((call, i) => ({ path: call[0] as string, order: writeSpy.mock.invocationCallOrder[i] }))
+      .filter((c) => c.path === BRAIN_FORMAT_PATH)
+    expect(stampOrders.length).toBe(1)
+    expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false)
+
+    // The derived index rebuilt, and the rebuild happened BEFORE the stamp
+    // (non-destructive ordering: the marker only advances once the rebuild that
+    // it certifies has run).
+    expect(rebuildSpy).toHaveBeenCalled()
+    const firstRebuildOrder = rebuildSpy.mock.invocationCallOrder[0]
+    expect(firstRebuildOrder).toBeLessThan(stampOrders[0].order)
+
+    await second.close()
+    brains.splice(brains.indexOf(second), 1)
+
+    // And the new marker is durable on disk at the EXPECTED epoch.
+    expect(await readMarkerFromDisk(dir)).toEqual(CURRENT_MARKER)
+  })
+
+  // (d) No marker (pre-handshake brain) → treated as stale → rebuild + stamp.
+  it('(d) no marker (pre-handshake brain): treated as stale → rebuilds and stamps the marker', async () => {
+    const dir = makeTempDir()
+    const first = await openFsBrain(dir)
+    await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } })
+    await first.close()
+    brains.splice(brains.indexOf(first), 1)
+
+    // A brain written before the handshake existed has NO marker on disk.
+    await seedMarkerOnDisk(dir, null)
+    expect(await readMarkerFromDisk(dir)).toBeNull()
+
+    const rebuildSpy = vi.spyOn(MetadataIndexManager.prototype, 'rebuild')
+    const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject')
+
+    const second = await openFsBrain(dir)
+
+    expect(rebuildSpy).toHaveBeenCalled()
+    const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH)
+    expect(stampWrites.length).toBe(1)
+    expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false)
+
+    await second.close()
+    brains.splice(brains.indexOf(second), 1)
+    expect(await readMarkerFromDisk(dir)).toEqual(CURRENT_MARKER)
+  })
+
+  // The epoch-drift trigger itself: a format change forces a rebuild past the
+  // warm-index fast path (the gap this feature closes); a current epoch does not.
+  it('epoch drift forces a rebuild of all three indexes past the warm fast path; current epoch does not', async () => {
+    const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    brains.push(brain)
+
+    // Warm the indexes (size() > 0) so the fast-path early return is live.
+    await brain.add({ data: 'a', type: NounType.Document, metadata: { k: 1 } })
+    await brain.add({ data: 'b', type: NounType.Document, metadata: { k: 2 } })
+    const internals = brain as unknown as {
+      index: { size(): number; rebuild(...a: unknown[]): Promise<unknown> }
+      metadataIndex: { rebuild(...a: unknown[]): Promise<unknown> }
+      graphIndex: { rebuild(...a: unknown[]): Promise<unknown> }
+      _indexEpochStale: boolean
+      rebuildIndexesIfNeeded(force?: boolean): Promise<void>
+    }
+    expect(internals.index.size()).toBeGreaterThan(0)
+
+    const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined)
+    const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined)
+    const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined)
+
+    // Current epoch: the warm fast path skips every rebuild.
+    internals._indexEpochStale = false
+    await internals.rebuildIndexesIfNeeded()
+    expect(idxSpy).not.toHaveBeenCalled()
+    expect(miSpy).not.toHaveBeenCalled()
+    expect(giSpy).not.toHaveBeenCalled()
+
+    // Drifted epoch: all three rebuild even though the indexes are non-empty.
+    internals._indexEpochStale = true
+    await internals.rebuildIndexesIfNeeded()
+    expect(idxSpy).toHaveBeenCalledTimes(1)
+    expect(miSpy).toHaveBeenCalledTimes(1)
+    expect(giSpy).toHaveBeenCalledTimes(1)
+    // The stamp cleared the stale flag after the verified rebuild.
+    expect(internals._indexEpochStale).toBe(false)
+  })
+
+  // (e) formatInfo() is synchronously available immediately after init (no await).
+  it('(e) formatInfo() is synchronously available immediately after init', async () => {
+    const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
+    await brain.init()
+    brains.push(brain)
+
+    // No await on the call itself — it is a pure in-memory constant read.
+    const info = brain.formatInfo()
+    expect(info).toEqual(CURRENT_MARKER)
+    expect(typeof info.indexEpoch).toBe('number')
+    expect(typeof info.dataFormat).toBe('string')
+  })
+})