soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): version-handshake marker (formatInfo + indexEpoch) for whole-brain auto-upgrade

Browse source 
Add _system/brain-format.json { dataFormat, indexEpoch } + a sync brain.formatInfo()
accessor + a compiled EXPECTED_INDEX_EPOCH, the surface a native provider reads at
init() to drive whole-brain auto-upgrade, and brainy's own derived-index
rebuild-on-format-drift trigger (closing the gap where JS indexes rebuilt only on
size()===0, never on a format-version change).

indexEpoch is shared and lockstep-bumped with the native provider on any coordinated
release whose on-disk derived-index format changes; dataFormat is brainy-owned. On open,
the marker is read in the store-open phase BEFORE provider construction (so formatInfo()
is synchronously available at the provider's init); a drifted or absent epoch rebuilds
the derived indexes from canonical records, and the marker is stamped only AFTER the
rebuild verifies (build-new -> verify -> stamp; a crash before the stamp re-triggers the
idempotent rebuild). brainFormat.ts is the single source of the shared constants.
6-case test; 1724 unit green.
This commit is contained in:
David Snelling 2026-06-30 09:53:10 -07:00
parent 229b0679fc
commit fc7f110479
3 changed files with 499 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

128
src/brainy.ts
View file

@ -17,6 +17,13 @@ import type { StorageOptions } from './storage/storageFactory.js'
import { rebuildCounts } from './utils/rebuildCounts.js'
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
import { BaseStorage } from './storage/baseStorage.js'
import {
CURRENT_DATA_FORMAT,
EXPECTED_INDEX_EPOCH,
readBrainFormat,
writeBrainFormat
} from './storage/brainFormat.js'
import type { BrainFormat } from './storage/brainFormat.js'
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
import {
@ -407,6 +414,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
private _graphAdjacencyVerified = false
/** Re-entrancy guard: a verify (rebuild → reads) is in flight. */
private _graphAdjacencyVerifying = false
/**
* 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat}
* marker (`_system/brain-format.json`) as read during the store-open phase,
* or `null` for a pre-handshake / brand-new brain. Populated BEFORE any
* derived index or native provider is constructed, so {@link formatInfo}
* answers synchronously when the provider reads it at its own init().
* Refreshed to the current marker once it is (re)stamped.
*/
private _brainFormat: BrainFormat | null = null
/**
* True when the on-disk {@link _brainFormat} is absent OR carries an
* `indexEpoch` different from {@link EXPECTED_INDEX_EPOCH} — i.e. the derived
* JS indexes on disk predate this build and must be rebuilt from the
* canonical records (the epoch-drift rebuild trigger that closes the gap
* where JS indexes rebuilt only on `size()===0`, never on a format change).
* Cleared once the rebuild verifies and the marker is re-stamped.
*/
private _indexEpochStale = false
// Sub-APIs (lazy-loaded)
private _nlp?: NaturalLanguageProcessor
@ -799,6 +824,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
readOnly: this.config.mode === 'reader'
})
// 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
// marker (`_system/brain-format.json`) into an in-memory field NOW —
// after the store-open phase, but BEFORE any derived index or native
// provider is constructed below — so `formatInfo()` is populated when the
// provider reads it synchronously at its own init(). A drifted or absent
// `indexEpoch` means the on-disk derived-index format predates this build
// (the derived JS indexes are stale); `rebuildIndexesIfNeeded()` rebuilds
// them from the canonical records and then re-stamps the marker AFTER the
// rebuild verifies (non-destructive: a crash mid-rebuild leaves the old /
// absent marker, so the next open idempotently re-rebuilds).
this._brainFormat = await readBrainFormat(this.storage)
this._indexEpochStale =
this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH
// Provider: embeddings (reassign embedder if plugin provides one)
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
if (embeddingProvider) {
@ -6339,6 +6378,35 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return this.generationStore.generation()
}
/**
* @description The data-layer + derived-index format this brain instance runs
* as — the SYNCHRONOUS half of the 7.x → 8.0 version handshake. Returns the
* compiled {@link CURRENT_DATA_FORMAT} / {@link EXPECTED_INDEX_EPOCH}
* constants (the single source of truth shared with the native provider, in
* `src/storage/brainFormat.ts`): after `init()` the brain has reconciled any
* on-disk drift and IS running the current format, so this reports what it
* runs as, not necessarily the (possibly older) marker it opened.
*
* This is a pure in-memory constant read — no `await`, no storage I/O — by
* design: a native metadata/index provider calls it synchronously at its own
* `init()` (which runs during this brain's provider-construction phase, after
* the marker has been loaded) to confirm "running data-format X, index epoch
* N" before binding its native readers.
*
* The on-disk marker (`_system/brain-format.json`) is reconciled separately:
* on open Brainy compares the on-disk `indexEpoch` against
* {@link EXPECTED_INDEX_EPOCH}; a mismatch or an absent marker rebuilds the
* derived indexes and re-stamps the marker (see `rebuildIndexesIfNeeded`).
*
* @returns `{ dataFormat, indexEpoch }` for the running build.
* @example
* const { dataFormat, indexEpoch } = brain.formatInfo()
* // dataFormat === '8.0', indexEpoch === 1
*/
formatInfo(): BrainFormat {
return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
}
/**
* @description Read the reified transaction log — one entry per committed
* generation, carrying the committed generation, the commit timestamp, and
@ -12946,8 +13014,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
// OPTIMIZATION: Instant check - if index already has data, skip immediately
// This gives 0s startup for warm restarts (vs 50-100ms of async checks)
if (this.index.size() > 0 && !force) {
// This gives 0s startup for warm restarts (vs 50-100ms of async checks).
// The epoch-drift guard suppresses this fast path: an index that loaded
// eagerly (size()>0) but whose on-disk format predates this build
// (`_indexEpochStale`) must still rebuild — that is the exact gap this
// handshake closes (rebuild on a format change, not only on size()===0).
if (this.index.size() > 0 && !force && !this._indexEpochStale) {
if (!this.config.silent) {
console.log(
`✅ Index already populated (${this.index.size().toLocaleString()} entities) - 0s startup!`
@ -12966,6 +13038,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (force && !this.config.silent) {
console.log('✅ Storage empty - no rebuild needed')
}
// A fresh / empty brain's (empty) derived indexes are trivially current
// for this build's epoch — stamp the version marker so the next open
// recognises it as current and skips the drift rebuild.
await this.stampBrainFormatIfNeeded()
return
}
@ -12988,10 +13064,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
graphIndexSize === 0 ||
(typeof graphIsReady.isReady === 'function' && !graphIsReady.isReady())
// Epoch-drift trigger: a format-version change makes EVERY derived index
// suspect even when each is non-empty, so it forces a rebuild of all
// three from the canonical records (not just the empty ones below).
const epochStale = this._indexEpochStale
const needsRebuild =
metadataStats.totalEntries === 0 ||
hnswIndexSize === 0 ||
needsGraphRebuild
needsGraphRebuild ||
epochStale
if (!needsRebuild && !force) {
// All indexes already populated, no rebuild needed
@ -13029,17 +13111,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints
// and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s
// ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()).
if (needsGraphRebuild) {
if (needsGraphRebuild || epochStale) {
await this.hydrateIdMapperForGraphRebuild()
}
// Rebuild all 3 indexes in parallel for performance
// Indexes load their data from storage (no recomputation)
// Indexes load their data from storage (no recomputation). An epoch drift
// (`epochStale`) rebuilds ALL three regardless of their current size —
// the on-disk format changed, so every derived index is rebuilt from the
// canonical records.
const rebuildStartTime = Date.now()
await Promise.all([
metadataStats.totalEntries === 0 ? this.metadataIndex.rebuild() : Promise.resolve(),
hnswIndexSize === 0 ? this.index.rebuild() : Promise.resolve(),
needsGraphRebuild ? this.graphIndex.rebuild() : Promise.resolve()
metadataStats.totalEntries === 0 || epochStale ? this.metadataIndex.rebuild() : Promise.resolve(),
hnswIndexSize === 0 || epochStale ? this.index.rebuild() : Promise.resolve(),
needsGraphRebuild || epochStale ? this.graphIndex.rebuild() : Promise.resolve()
])
const rebuildDuration = Date.now() - rebuildStartTime
@ -13066,6 +13151,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
}
// 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
// have now rebuilt and verified, so they match this build's epoch —
// re-stamp the marker LAST, only here. A crash anywhere above leaves the
// old / absent marker, so the next open re-detects the drift and re-runs
// the (idempotent) rebuild; the marker is never advanced ahead of the
// indexes it certifies. A no-op when the epoch was already current.
await this.stampBrainFormatIfNeeded()
} catch (error) {
// A storage READ failure here is surfaced by getNouns/getVerbs as a named
// BrainyError — it means we could not even read the store to decide whether
@ -13083,6 +13176,25 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
/**
* @description Re-stamp the 8.0 ⇄ native-provider version-handshake marker
* (`_system/brain-format.json`) to this build's {@link CURRENT_DATA_FORMAT} /
* {@link EXPECTED_INDEX_EPOCH} — but ONLY when the on-disk marker was absent
* or carried a drifted epoch ({@link _indexEpochStale}). Called at the
* verified-completion points of {@link rebuildIndexesIfNeeded} (after the
* derived-index rebuild, or on the empty-storage fast path), so the marker is
* never advanced ahead of the indexes it certifies — a crash before this
* point re-triggers the idempotent rebuild on the next open. Readers never
* write, so this is a no-op in reader mode.
*/
private async stampBrainFormatIfNeeded(): Promise<void> {
if (this.isReadOnly) return
if (!this._indexEpochStale) return
await writeBrainFormat(this.storage)
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
this._indexEpochStale = false
}
/**
* Check health of metadata indexes
*