open-brainy/src/storage/brainFormat.ts

/**
 * @module storage/brainFormat
 * @description The 7.x → 8.0 version-handshake marker: the small persisted
 * artifact at `_system/brain-format.json` that lets Brainy and a native
 * metadata/index provider agree, at open time, on the on-disk format of the
 * brain AND of its derived indexes.
 *
 * Two fields, two owners:
 *
 * - `dataFormat` — **Brainy-owned**. The data-layer version string (the
 *   canonical entity / relationship / generation-record layout). Brainy bumps
 *   it when that data layout changes; a native provider reads it only to
 *   confirm the major line it is binding against (e.g. "this is an 8.0 brain").
 * - `indexEpoch` — **SHARED**. A monotonic integer that Brainy and the native
 *   provider bump TOGETHER, in lockstep, on any coordinated release where the
 *   on-disk format of ANY derived index (the HNSW vectors, the metadata
 *   postings, or the graph adjacency) changes. It is the single switch that
 *   declares "every derived index written by an older build is stale and must
 *   be rebuilt from the canonical records."
 *
 * Open-time handshake. On open, Brainy compares the on-disk `indexEpoch`
 * against the compiled {@link EXPECTED_INDEX_EPOCH}. A mismatch — OR an absent
 * marker (a pre-handshake brain, or a brand-new store) — means the derived
 * indexes on disk predate this build, so Brainy rebuilds them from the
 * canonical records and only THEN re-stamps the marker. The provider reads the
 * same surface synchronously through `brain.formatInfo()` at its own provider
 * init() to confirm "running data-format X, index epoch N" before binding its
 * native readers.
 *
 * Lockstep contract. NEVER bump {@link EXPECTED_INDEX_EPOCH} unilaterally. It
 * advances only on a coordinated release whose on-disk derived-index format
 * actually changed, and both projects ship the SAME new value in the same
 * release — so a brain written by either side is recognised as current by the
 * other, and a brain written by an older build of either side is recognised as
 * stale and rebuilt. The constant living here makes this module the single
 * source of truth both sides reference.
 *
 * Non-destructive stamping. The marker is the LAST thing written, AFTER the
 * rebuild has verified. A crash between the rebuild and the stamp leaves the
 * old / absent marker on disk, so the next open re-detects the drift and
 * re-runs the (idempotent) rebuild — the marker is never advanced ahead of the
 * indexes it certifies. This mirrors the generational record layer's
 * "build-new → verify → atomic-rename" commit discipline
 * (`src/db/generationStore.ts`).
 */

/**
 * @description The narrow storage surface the marker helpers need — the
 * raw-object read/write primitives every `BaseStorage` adapter implements (the
 * same surface the generational record layer uses for `_system/` artifacts).
 * Declared structurally so this module carries no runtime dependency on the
 * adapter class.
 */
export interface BrainFormatStorage {
  /** Read a raw object at a storage-root-relative path (`null` if absent). */
  readRawObject(path: string): Promise<unknown | null>
  /** Write a raw object at a storage-root-relative path (atomic tmp+rename on disk). */
  writeRawObject(path: string, data: unknown): Promise<void>
}

/** Storage-root-relative path of the version-handshake marker. */
export const BRAIN_FORMAT_PATH = '_system/brain-format.json'

/**
 * @description The compiled derived-index format epoch this build expects on
 * disk. SHARED and lockstep-bumped with the native provider: advance it (in
 * BOTH projects, to the same value, in a coordinated release) on any change to
 * the on-disk format of any derived index — never on one side alone. Start = 1
 * (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an
 * absent marker — triggers a full derived-index rebuild on open.
 */
// Epoch 3 (2026-08-03, the namespace-law pair): the index key format split
// the two namespaces — user fields keep bare flattened keys, the ten system
// scalars moved to literal 'system.<field>' keys (the legacy 'noun' column
// spelling died with them). Every brain rebuilds its derived indexes from
// canonical at first open onto the frozen keys.
// Epoch 2 (2026-08-03, same day, the interim pair): user metadata fields
// named `level` became indexable on both engines; poisoned multi-valued
// `level` columns healed through the rebuild.
export const EXPECTED_INDEX_EPOCH = 3

/**
 * @description The data-layer format string this build writes and runs as.
 * Brainy-owned; bumped when the canonical data layout changes. Start = '8.0'.
 */
export const CURRENT_DATA_FORMAT = '8.0'

/**
 * @description The persisted shape of `_system/brain-format.json` — the
 * value `brain.formatInfo()` returns for the running brain, and the value
 * read back from disk to drive the epoch-drift rebuild trigger.
 */
export interface BrainFormat {
  /** Brainy-owned data-layer version string (e.g. `'8.0'`). */
  dataFormat: string
  /** Shared, lockstep-bumped derived-index format epoch (e.g. `1`). */
  indexEpoch: number
}

/**
 * @description Read the on-disk version-handshake marker, or `null` when it is
 * absent (a pre-handshake brain, or a brand-new store). A malformed marker
 * (not an object, missing either field, or a non-finite epoch) is also treated
 * as `null` — a corrupt marker forces a safe rebuild rather than trusting a bad
 * epoch.
 * @param storage - The brain's storage adapter (raw-object surface).
 * @returns The parsed {@link BrainFormat}, or `null`.
 * @example
 * const onDisk = await readBrainFormat(brain.storage)
 * const stale = onDisk === null || onDisk.indexEpoch !== EXPECTED_INDEX_EPOCH
 */
export async function readBrainFormat(
  storage: Pick<BrainFormatStorage, 'readRawObject'>
): Promise<BrainFormat | null> {
  const raw = (await storage.readRawObject(BRAIN_FORMAT_PATH)) as Partial<BrainFormat> | null
  if (raw === null || typeof raw !== 'object') return null
  if (
    typeof raw.dataFormat !== 'string' ||
    typeof raw.indexEpoch !== 'number' ||
    !Number.isFinite(raw.indexEpoch)
  ) {
    return null
  }
  return { dataFormat: raw.dataFormat, indexEpoch: raw.indexEpoch }
}

/**
 * @description Stamp this build's {@link CURRENT_DATA_FORMAT} /
 * {@link EXPECTED_INDEX_EPOCH} to `_system/brain-format.json` (an atomic
 * tmp+rename on the filesystem adapter). MUST be called only AFTER the
 * derived-index rebuild has verified: the marker certifies the indexes on
 * disk, so advancing it ahead of them would suppress the rebuild a future open
 * needs (the non-destructive contract — see the module docs).
 * @param storage - The brain's storage adapter (raw-object surface).
 */
export async function writeBrainFormat(
  storage: Pick<BrainFormatStorage, 'writeRawObject'>
): Promise<void> {
  const marker: BrainFormat = {
    dataFormat: CURRENT_DATA_FORMAT,
    indexEpoch: EXPECTED_INDEX_EPOCH
  }
  await storage.writeRawObject(BRAIN_FORMAT_PATH, marker)
}