soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): version-handshake marker (formatInfo + indexEpoch) for whole-brain auto-upgrade

Browse source 
Add _system/brain-format.json { dataFormat, indexEpoch } + a sync brain.formatInfo()
accessor + a compiled EXPECTED_INDEX_EPOCH, the surface a native provider reads at
init() to drive whole-brain auto-upgrade, and brainy's own derived-index
rebuild-on-format-drift trigger (closing the gap where JS indexes rebuilt only on
size()===0, never on a format-version change).

indexEpoch is shared and lockstep-bumped with the native provider on any coordinated
release whose on-disk derived-index format changes; dataFormat is brainy-owned. On open,
the marker is read in the store-open phase BEFORE provider construction (so formatInfo()
is synchronously available at the provider's init); a drifted or absent epoch rebuilds
the derived indexes from canonical records, and the marker is stamped only AFTER the
rebuild verifies (build-new -> verify -> stamp; a crash before the stamp re-triggers the
idempotent rebuild). brainFormat.ts is the single source of the shared constants.
6-case test; 1724 unit green.
This commit is contained in:
David Snelling 2026-06-30 09:53:10 -07:00
parent 229b0679fc
commit fc7f110479
3 changed files with 499 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

128
src/brainy.ts
View file

@ -17,6 +17,13 @@ import type { StorageOptions } from './storage/storageFactory.js'
import { rebuildCounts } from './utils/rebuildCounts.js'
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
import { BaseStorage } from './storage/baseStorage.js'
import {
CURRENT_DATA_FORMAT,
EXPECTED_INDEX_EPOCH,
readBrainFormat,
writeBrainFormat
} from './storage/brainFormat.js'
import type { BrainFormat } from './storage/brainFormat.js'
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
import {
@ -407,6 +414,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
private _graphAdjacencyVerified = false
/** Re-entrancy guard: a verify (rebuild → reads) is in flight. */
private _graphAdjacencyVerifying = false
/**
* 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat}
* marker (`_system/brain-format.json`) as read during the store-open phase,
* or `null` for a pre-handshake / brand-new brain. Populated BEFORE any
* derived index or native provider is constructed, so {@link formatInfo}
* answers synchronously when the provider reads it at its own init().
* Refreshed to the current marker once it is (re)stamped.
*/
private _brainFormat: BrainFormat | null = null
/**
* True when the on-disk {@link _brainFormat} is absent OR carries an
* `indexEpoch` different from {@link EXPECTED_INDEX_EPOCH} — i.e. the derived
* JS indexes on disk predate this build and must be rebuilt from the
* canonical records (the epoch-drift rebuild trigger that closes the gap
* where JS indexes rebuilt only on `size()===0`, never on a format change).
* Cleared once the rebuild verifies and the marker is re-stamped.
*/
private _indexEpochStale = false
// Sub-APIs (lazy-loaded)
private _nlp?: NaturalLanguageProcessor
@ -799,6 +824,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
readOnly: this.config.mode === 'reader'
})
// 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
// marker (`_system/brain-format.json`) into an in-memory field NOW —
// after the store-open phase, but BEFORE any derived index or native
// provider is constructed below — so `formatInfo()` is populated when the
// provider reads it synchronously at its own init(). A drifted or absent
// `indexEpoch` means the on-disk derived-index format predates this build
// (the derived JS indexes are stale); `rebuildIndexesIfNeeded()` rebuilds
// them from the canonical records and then re-stamps the marker AFTER the
// rebuild verifies (non-destructive: a crash mid-rebuild leaves the old /
// absent marker, so the next open idempotently re-rebuilds).
this._brainFormat = await readBrainFormat(this.storage)
this._indexEpochStale =
this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH
// Provider: embeddings (reassign embedder if plugin provides one)
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
if (embeddingProvider) {
@ -6339,6 +6378,35 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return this.generationStore.generation()
}
/**
* @description The data-layer + derived-index format this brain instance runs
* as — the SYNCHRONOUS half of the 7.x → 8.0 version handshake. Returns the
* compiled {@link CURRENT_DATA_FORMAT} / {@link EXPECTED_INDEX_EPOCH}
* constants (the single source of truth shared with the native provider, in
* `src/storage/brainFormat.ts`): after `init()` the brain has reconciled any
* on-disk drift and IS running the current format, so this reports what it
* runs as, not necessarily the (possibly older) marker it opened.
*
* This is a pure in-memory constant read — no `await`, no storage I/O — by
* design: a native metadata/index provider calls it synchronously at its own
* `init()` (which runs during this brain's provider-construction phase, after
* the marker has been loaded) to confirm "running data-format X, index epoch
* N" before binding its native readers.
*
* The on-disk marker (`_system/brain-format.json`) is reconciled separately:
* on open Brainy compares the on-disk `indexEpoch` against
* {@link EXPECTED_INDEX_EPOCH}; a mismatch or an absent marker rebuilds the
* derived indexes and re-stamps the marker (see `rebuildIndexesIfNeeded`).
*
* @returns `{ dataFormat, indexEpoch }` for the running build.
* @example
* const { dataFormat, indexEpoch } = brain.formatInfo()
* // dataFormat === '8.0', indexEpoch === 1
*/
formatInfo(): BrainFormat {
return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
}
/**
* @description Read the reified transaction log — one entry per committed
* generation, carrying the committed generation, the commit timestamp, and
@ -12946,8 +13014,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
// OPTIMIZATION: Instant check - if index already has data, skip immediately
// This gives 0s startup for warm restarts (vs 50-100ms of async checks)
if (this.index.size() > 0 && !force) {
// This gives 0s startup for warm restarts (vs 50-100ms of async checks).
// The epoch-drift guard suppresses this fast path: an index that loaded
// eagerly (size()>0) but whose on-disk format predates this build
// (`_indexEpochStale`) must still rebuild — that is the exact gap this
// handshake closes (rebuild on a format change, not only on size()===0).
if (this.index.size() > 0 && !force && !this._indexEpochStale) {
if (!this.config.silent) {
console.log(
`✅ Index already populated (${this.index.size().toLocaleString()} entities) - 0s startup!`
@ -12966,6 +13038,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (force && !this.config.silent) {
console.log('✅ Storage empty - no rebuild needed')
}
// A fresh / empty brain's (empty) derived indexes are trivially current
// for this build's epoch — stamp the version marker so the next open
// recognises it as current and skips the drift rebuild.
await this.stampBrainFormatIfNeeded()
return
}
@ -12988,10 +13064,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
graphIndexSize === 0 ||
(typeof graphIsReady.isReady === 'function' && !graphIsReady.isReady())
// Epoch-drift trigger: a format-version change makes EVERY derived index
// suspect even when each is non-empty, so it forces a rebuild of all
// three from the canonical records (not just the empty ones below).
const epochStale = this._indexEpochStale
const needsRebuild =
metadataStats.totalEntries === 0 ||
hnswIndexSize === 0 ||
needsGraphRebuild
needsGraphRebuild ||
epochStale
if (!needsRebuild && !force) {
// All indexes already populated, no rebuild needed
@ -13029,17 +13111,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints
// and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s
// ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()).
if (needsGraphRebuild) {
if (needsGraphRebuild || epochStale) {
await this.hydrateIdMapperForGraphRebuild()
}
// Rebuild all 3 indexes in parallel for performance
// Indexes load their data from storage (no recomputation)
// Indexes load their data from storage (no recomputation). An epoch drift
// (`epochStale`) rebuilds ALL three regardless of their current size —
// the on-disk format changed, so every derived index is rebuilt from the
// canonical records.
const rebuildStartTime = Date.now()
await Promise.all([
metadataStats.totalEntries === 0 ? this.metadataIndex.rebuild() : Promise.resolve(),
hnswIndexSize === 0 ? this.index.rebuild() : Promise.resolve(),
needsGraphRebuild ? this.graphIndex.rebuild() : Promise.resolve()
metadataStats.totalEntries === 0 || epochStale ? this.metadataIndex.rebuild() : Promise.resolve(),
hnswIndexSize === 0 || epochStale ? this.index.rebuild() : Promise.resolve(),
needsGraphRebuild || epochStale ? this.graphIndex.rebuild() : Promise.resolve()
])
const rebuildDuration = Date.now() - rebuildStartTime
@ -13066,6 +13151,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
}
// 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
// have now rebuilt and verified, so they match this build's epoch —
// re-stamp the marker LAST, only here. A crash anywhere above leaves the
// old / absent marker, so the next open re-detects the drift and re-runs
// the (idempotent) rebuild; the marker is never advanced ahead of the
// indexes it certifies. A no-op when the epoch was already current.
await this.stampBrainFormatIfNeeded()
} catch (error) {
// A storage READ failure here is surfaced by getNouns/getVerbs as a named
// BrainyError — it means we could not even read the store to decide whether
@ -13083,6 +13176,25 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
/**
* @description Re-stamp the 8.0 ⇄ native-provider version-handshake marker
* (`_system/brain-format.json`) to this build's {@link CURRENT_DATA_FORMAT} /
* {@link EXPECTED_INDEX_EPOCH} — but ONLY when the on-disk marker was absent
* or carried a drifted epoch ({@link _indexEpochStale}). Called at the
* verified-completion points of {@link rebuildIndexesIfNeeded} (after the
* derived-index rebuild, or on the empty-storage fast path), so the marker is
* never advanced ahead of the indexes it certifies — a crash before this
* point re-triggers the idempotent rebuild on the next open. Readers never
* write, so this is a no-op in reader mode.
*/
private async stampBrainFormatIfNeeded(): Promise<void> {
if (this.isReadOnly) return
if (!this._indexEpochStale) return
await writeBrainFormat(this.storage)
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
this._indexEpochStale = false
}
/**
* Check health of metadata indexes
*

136
src/storage/brainFormat.ts Normal file
View file

@ -0,0 +1,136 @@
/**
* @module storage/brainFormat
* @description The 7.x → 8.0 version-handshake marker: the small persisted
* artifact at `_system/brain-format.json` that lets Brainy and a native
* metadata/index provider agree, at open time, on the on-disk format of the
* brain AND of its derived indexes.
*
* Two fields, two owners:
*
* - `dataFormat` — **Brainy-owned**. The data-layer version string (the
* canonical entity / relationship / generation-record layout). Brainy bumps
* it when that data layout changes; a native provider reads it only to
* confirm the major line it is binding against (e.g. "this is an 8.0 brain").
* - `indexEpoch` — **SHARED**. A monotonic integer that Brainy and the native
* provider bump TOGETHER, in lockstep, on any coordinated release where the
* on-disk format of ANY derived index (the HNSW vectors, the metadata
* postings, or the graph adjacency) changes. It is the single switch that
* declares "every derived index written by an older build is stale and must
* be rebuilt from the canonical records."
*
* Open-time handshake. On open, Brainy compares the on-disk `indexEpoch`
* against the compiled {@link EXPECTED_INDEX_EPOCH}. A mismatch — OR an absent
* marker (a pre-handshake brain, or a brand-new store) — means the derived
* indexes on disk predate this build, so Brainy rebuilds them from the
* canonical records and only THEN re-stamps the marker. The provider reads the
* same surface synchronously through `brain.formatInfo()` at its own provider
* init() to confirm "running data-format X, index epoch N" before binding its
* native readers.
*
* Lockstep contract. NEVER bump {@link EXPECTED_INDEX_EPOCH} unilaterally. It
* advances only on a coordinated release whose on-disk derived-index format
* actually changed, and both projects ship the SAME new value in the same
* release — so a brain written by either side is recognised as current by the
* other, and a brain written by an older build of either side is recognised as
* stale and rebuilt. The constant living here makes this module the single
* source of truth both sides reference.
*
* Non-destructive stamping. The marker is the LAST thing written, AFTER the
* rebuild has verified. A crash between the rebuild and the stamp leaves the
* old / absent marker on disk, so the next open re-detects the drift and
* re-runs the (idempotent) rebuild — the marker is never advanced ahead of the
* indexes it certifies. This mirrors the generational record layer's
* "build-new → verify → atomic-rename" commit discipline
* (`src/db/generationStore.ts`).
*/
/**
* @description The narrow storage surface the marker helpers need — the
* raw-object read/write primitives every `BaseStorage` adapter implements (the
* same surface the generational record layer uses for `_system/` artifacts).
* Declared structurally so this module carries no runtime dependency on the
* adapter class.
*/
export interface BrainFormatStorage {
/** Read a raw object at a storage-root-relative path (`null` if absent). */
readRawObject(path: string): Promise<unknown | null>
/** Write a raw object at a storage-root-relative path (atomic tmp+rename on disk). */
writeRawObject(path: string, data: unknown): Promise<void>
}
/** Storage-root-relative path of the version-handshake marker. */
export const BRAIN_FORMAT_PATH = '_system/brain-format.json'
/**
* @description The compiled derived-index format epoch this build expects on
* disk. SHARED and lockstep-bumped with the native provider: advance it (in
* BOTH projects, to the same value, in a coordinated release) on any change to
* the on-disk format of any derived index — never on one side alone. Start = 1
* (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an
* absent marker — triggers a full derived-index rebuild on open.
*/
export const EXPECTED_INDEX_EPOCH = 1
/**
* @description The data-layer format string this build writes and runs as.
* Brainy-owned; bumped when the canonical data layout changes. Start = '8.0'.
*/
export const CURRENT_DATA_FORMAT = '8.0'
/**
* @description The persisted shape of `_system/brain-format.json` — the
* value `brain.formatInfo()` returns for the running brain, and the value
* read back from disk to drive the epoch-drift rebuild trigger.
*/
export interface BrainFormat {
/** Brainy-owned data-layer version string (e.g. `'8.0'`). */
dataFormat: string
/** Shared, lockstep-bumped derived-index format epoch (e.g. `1`). */
indexEpoch: number
}
/**
* @description Read the on-disk version-handshake marker, or `null` when it is
* absent (a pre-handshake brain, or a brand-new store). A malformed marker
* (not an object, missing either field, or a non-finite epoch) is also treated
* as `null` — a corrupt marker forces a safe rebuild rather than trusting a bad
* epoch.
* @param storage - The brain's storage adapter (raw-object surface).
* @returns The parsed {@link BrainFormat}, or `null`.
* @example
* const onDisk = await readBrainFormat(brain.storage)
* const stale = onDisk === null || onDisk.indexEpoch !== EXPECTED_INDEX_EPOCH
*/
export async function readBrainFormat(
storage: Pick<BrainFormatStorage, 'readRawObject'>
): Promise<BrainFormat | null> {
const raw = (await storage.readRawObject(BRAIN_FORMAT_PATH)) as Partial<BrainFormat> | null
if (raw === null || typeof raw !== 'object') return null
if (
typeof raw.dataFormat !== 'string' ||
typeof raw.indexEpoch !== 'number' ||
!Number.isFinite(raw.indexEpoch)
) {
return null
}
return { dataFormat: raw.dataFormat, indexEpoch: raw.indexEpoch }
}
/**
* @description Stamp this build's {@link CURRENT_DATA_FORMAT} /
* {@link EXPECTED_INDEX_EPOCH} to `_system/brain-format.json` (an atomic
* tmp+rename on the filesystem adapter). MUST be called only AFTER the
* derived-index rebuild has verified: the marker certifies the indexes on
* disk, so advancing it ahead of them would suppress the rebuild a future open
* needs (the non-destructive contract — see the module docs).
* @param storage - The brain's storage adapter (raw-object surface).
*/
export async function writeBrainFormat(
storage: Pick<BrainFormatStorage, 'writeRawObject'>
): Promise<void> {
const marker: BrainFormat = {
dataFormat: CURRENT_DATA_FORMAT,
indexEpoch: EXPECTED_INDEX_EPOCH
}
await storage.writeRawObject(BRAIN_FORMAT_PATH, marker)
}