feat(8.0): version-handshake marker (formatInfo + indexEpoch) for whole-brain auto-upgrade
Add _system/brain-format.json { dataFormat, indexEpoch } + a sync brain.formatInfo()
accessor + a compiled EXPECTED_INDEX_EPOCH, the surface a native provider reads at
init() to drive whole-brain auto-upgrade, and brainy's own derived-index
rebuild-on-format-drift trigger (closing the gap where JS indexes rebuilt only on
size()===0, never on a format-version change).
indexEpoch is shared and lockstep-bumped with the native provider on any coordinated
release whose on-disk derived-index format changes; dataFormat is brainy-owned. On open,
the marker is read in the store-open phase BEFORE provider construction (so formatInfo()
is synchronously available at the provider's init); a drifted or absent epoch rebuilds
the derived indexes from canonical records, and the marker is stamped only AFTER the
rebuild verifies (build-new -> verify -> stamp; a crash before the stamp re-triggers the
idempotent rebuild). brainFormat.ts is the single source of the shared constants.
6-case test; 1724 unit green.
This commit is contained in:
parent
229b0679fc
commit
fc7f110479
3 changed files with 499 additions and 8 deletions
128
src/brainy.ts
128
src/brainy.ts
|
|
@ -17,6 +17,13 @@ import type { StorageOptions } from './storage/storageFactory.js'
|
||||||
import { rebuildCounts } from './utils/rebuildCounts.js'
|
import { rebuildCounts } from './utils/rebuildCounts.js'
|
||||||
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
|
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
|
||||||
import { BaseStorage } from './storage/baseStorage.js'
|
import { BaseStorage } from './storage/baseStorage.js'
|
||||||
|
import {
|
||||||
|
CURRENT_DATA_FORMAT,
|
||||||
|
EXPECTED_INDEX_EPOCH,
|
||||||
|
readBrainFormat,
|
||||||
|
writeBrainFormat
|
||||||
|
} from './storage/brainFormat.js'
|
||||||
|
import type { BrainFormat } from './storage/brainFormat.js'
|
||||||
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
|
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
|
||||||
import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
|
import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
|
||||||
import {
|
import {
|
||||||
|
|
@ -407,6 +414,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
private _graphAdjacencyVerified = false
|
private _graphAdjacencyVerified = false
|
||||||
/** Re-entrancy guard: a verify (rebuild → reads) is in flight. */
|
/** Re-entrancy guard: a verify (rebuild → reads) is in flight. */
|
||||||
private _graphAdjacencyVerifying = false
|
private _graphAdjacencyVerifying = false
|
||||||
|
/**
|
||||||
|
* 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat}
|
||||||
|
* marker (`_system/brain-format.json`) as read during the store-open phase,
|
||||||
|
* or `null` for a pre-handshake / brand-new brain. Populated BEFORE any
|
||||||
|
* derived index or native provider is constructed, so {@link formatInfo}
|
||||||
|
* answers synchronously when the provider reads it at its own init().
|
||||||
|
* Refreshed to the current marker once it is (re)stamped.
|
||||||
|
*/
|
||||||
|
private _brainFormat: BrainFormat | null = null
|
||||||
|
/**
|
||||||
|
* True when the on-disk {@link _brainFormat} is absent OR carries an
|
||||||
|
* `indexEpoch` different from {@link EXPECTED_INDEX_EPOCH} — i.e. the derived
|
||||||
|
* JS indexes on disk predate this build and must be rebuilt from the
|
||||||
|
* canonical records (the epoch-drift rebuild trigger that closes the gap
|
||||||
|
* where JS indexes rebuilt only on `size()===0`, never on a format change).
|
||||||
|
* Cleared once the rebuild verifies and the marker is re-stamped.
|
||||||
|
*/
|
||||||
|
private _indexEpochStale = false
|
||||||
|
|
||||||
// Sub-APIs (lazy-loaded)
|
// Sub-APIs (lazy-loaded)
|
||||||
private _nlp?: NaturalLanguageProcessor
|
private _nlp?: NaturalLanguageProcessor
|
||||||
|
|
@ -799,6 +824,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
readOnly: this.config.mode === 'reader'
|
readOnly: this.config.mode === 'reader'
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// 8.0 ⇄ native-provider version handshake: load the on-disk brain-format
|
||||||
|
// marker (`_system/brain-format.json`) into an in-memory field NOW —
|
||||||
|
// after the store-open phase, but BEFORE any derived index or native
|
||||||
|
// provider is constructed below — so `formatInfo()` is populated when the
|
||||||
|
// provider reads it synchronously at its own init(). A drifted or absent
|
||||||
|
// `indexEpoch` means the on-disk derived-index format predates this build
|
||||||
|
// (the derived JS indexes are stale); `rebuildIndexesIfNeeded()` rebuilds
|
||||||
|
// them from the canonical records and then re-stamps the marker AFTER the
|
||||||
|
// rebuild verifies (non-destructive: a crash mid-rebuild leaves the old /
|
||||||
|
// absent marker, so the next open idempotently re-rebuilds).
|
||||||
|
this._brainFormat = await readBrainFormat(this.storage)
|
||||||
|
this._indexEpochStale =
|
||||||
|
this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH
|
||||||
|
|
||||||
// Provider: embeddings (reassign embedder if plugin provides one)
|
// Provider: embeddings (reassign embedder if plugin provides one)
|
||||||
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
|
const embeddingProvider = this.pluginRegistry.getProvider<EmbeddingFunction>('embeddings')
|
||||||
if (embeddingProvider) {
|
if (embeddingProvider) {
|
||||||
|
|
@ -6339,6 +6378,35 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
return this.generationStore.generation()
|
return this.generationStore.generation()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The data-layer + derived-index format this brain instance runs
|
||||||
|
* as — the SYNCHRONOUS half of the 7.x → 8.0 version handshake. Returns the
|
||||||
|
* compiled {@link CURRENT_DATA_FORMAT} / {@link EXPECTED_INDEX_EPOCH}
|
||||||
|
* constants (the single source of truth shared with the native provider, in
|
||||||
|
* `src/storage/brainFormat.ts`): after `init()` the brain has reconciled any
|
||||||
|
* on-disk drift and IS running the current format, so this reports what it
|
||||||
|
* runs as, not necessarily the (possibly older) marker it opened.
|
||||||
|
*
|
||||||
|
* This is a pure in-memory constant read — no `await`, no storage I/O — by
|
||||||
|
* design: a native metadata/index provider calls it synchronously at its own
|
||||||
|
* `init()` (which runs during this brain's provider-construction phase, after
|
||||||
|
* the marker has been loaded) to confirm "running data-format X, index epoch
|
||||||
|
* N" before binding its native readers.
|
||||||
|
*
|
||||||
|
* The on-disk marker (`_system/brain-format.json`) is reconciled separately:
|
||||||
|
* on open Brainy compares the on-disk `indexEpoch` against
|
||||||
|
* {@link EXPECTED_INDEX_EPOCH}; a mismatch or an absent marker rebuilds the
|
||||||
|
* derived indexes and re-stamps the marker (see `rebuildIndexesIfNeeded`).
|
||||||
|
*
|
||||||
|
* @returns `{ dataFormat, indexEpoch }` for the running build.
|
||||||
|
* @example
|
||||||
|
* const { dataFormat, indexEpoch } = brain.formatInfo()
|
||||||
|
* // dataFormat === '8.0', indexEpoch === 1
|
||||||
|
*/
|
||||||
|
formatInfo(): BrainFormat {
|
||||||
|
return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Read the reified transaction log — one entry per committed
|
* @description Read the reified transaction log — one entry per committed
|
||||||
* generation, carrying the committed generation, the commit timestamp, and
|
* generation, carrying the committed generation, the commit timestamp, and
|
||||||
|
|
@ -12946,8 +13014,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
// OPTIMIZATION: Instant check - if index already has data, skip immediately
|
// OPTIMIZATION: Instant check - if index already has data, skip immediately
|
||||||
// This gives 0s startup for warm restarts (vs 50-100ms of async checks)
|
// This gives 0s startup for warm restarts (vs 50-100ms of async checks).
|
||||||
if (this.index.size() > 0 && !force) {
|
// The epoch-drift guard suppresses this fast path: an index that loaded
|
||||||
|
// eagerly (size()>0) but whose on-disk format predates this build
|
||||||
|
// (`_indexEpochStale`) must still rebuild — that is the exact gap this
|
||||||
|
// handshake closes (rebuild on a format change, not only on size()===0).
|
||||||
|
if (this.index.size() > 0 && !force && !this._indexEpochStale) {
|
||||||
if (!this.config.silent) {
|
if (!this.config.silent) {
|
||||||
console.log(
|
console.log(
|
||||||
`✅ Index already populated (${this.index.size().toLocaleString()} entities) - 0s startup!`
|
`✅ Index already populated (${this.index.size().toLocaleString()} entities) - 0s startup!`
|
||||||
|
|
@ -12966,6 +13038,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
if (force && !this.config.silent) {
|
if (force && !this.config.silent) {
|
||||||
console.log('✅ Storage empty - no rebuild needed')
|
console.log('✅ Storage empty - no rebuild needed')
|
||||||
}
|
}
|
||||||
|
// A fresh / empty brain's (empty) derived indexes are trivially current
|
||||||
|
// for this build's epoch — stamp the version marker so the next open
|
||||||
|
// recognises it as current and skips the drift rebuild.
|
||||||
|
await this.stampBrainFormatIfNeeded()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -12988,10 +13064,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
graphIndexSize === 0 ||
|
graphIndexSize === 0 ||
|
||||||
(typeof graphIsReady.isReady === 'function' && !graphIsReady.isReady())
|
(typeof graphIsReady.isReady === 'function' && !graphIsReady.isReady())
|
||||||
|
|
||||||
|
// Epoch-drift trigger: a format-version change makes EVERY derived index
|
||||||
|
// suspect even when each is non-empty, so it forces a rebuild of all
|
||||||
|
// three from the canonical records (not just the empty ones below).
|
||||||
|
const epochStale = this._indexEpochStale
|
||||||
|
|
||||||
const needsRebuild =
|
const needsRebuild =
|
||||||
metadataStats.totalEntries === 0 ||
|
metadataStats.totalEntries === 0 ||
|
||||||
hnswIndexSize === 0 ||
|
hnswIndexSize === 0 ||
|
||||||
needsGraphRebuild
|
needsGraphRebuild ||
|
||||||
|
epochStale
|
||||||
|
|
||||||
if (!needsRebuild && !force) {
|
if (!needsRebuild && !force) {
|
||||||
// All indexes already populated, no rebuild needed
|
// All indexes already populated, no rebuild needed
|
||||||
|
|
@ -13029,17 +13111,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints
|
// graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints
|
||||||
// and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s
|
// and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s
|
||||||
// ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()).
|
// ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()).
|
||||||
if (needsGraphRebuild) {
|
if (needsGraphRebuild || epochStale) {
|
||||||
await this.hydrateIdMapperForGraphRebuild()
|
await this.hydrateIdMapperForGraphRebuild()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rebuild all 3 indexes in parallel for performance
|
// Rebuild all 3 indexes in parallel for performance
|
||||||
// Indexes load their data from storage (no recomputation)
|
// Indexes load their data from storage (no recomputation). An epoch drift
|
||||||
|
// (`epochStale`) rebuilds ALL three regardless of their current size —
|
||||||
|
// the on-disk format changed, so every derived index is rebuilt from the
|
||||||
|
// canonical records.
|
||||||
const rebuildStartTime = Date.now()
|
const rebuildStartTime = Date.now()
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
metadataStats.totalEntries === 0 ? this.metadataIndex.rebuild() : Promise.resolve(),
|
metadataStats.totalEntries === 0 || epochStale ? this.metadataIndex.rebuild() : Promise.resolve(),
|
||||||
hnswIndexSize === 0 ? this.index.rebuild() : Promise.resolve(),
|
hnswIndexSize === 0 || epochStale ? this.index.rebuild() : Promise.resolve(),
|
||||||
needsGraphRebuild ? this.graphIndex.rebuild() : Promise.resolve()
|
needsGraphRebuild || epochStale ? this.graphIndex.rebuild() : Promise.resolve()
|
||||||
])
|
])
|
||||||
|
|
||||||
const rebuildDuration = Date.now() - rebuildStartTime
|
const rebuildDuration = Date.now() - rebuildStartTime
|
||||||
|
|
@ -13066,6 +13151,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
|
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
|
||||||
|
// have now rebuilt and verified, so they match this build's epoch —
|
||||||
|
// re-stamp the marker LAST, only here. A crash anywhere above leaves the
|
||||||
|
// old / absent marker, so the next open re-detects the drift and re-runs
|
||||||
|
// the (idempotent) rebuild; the marker is never advanced ahead of the
|
||||||
|
// indexes it certifies. A no-op when the epoch was already current.
|
||||||
|
await this.stampBrainFormatIfNeeded()
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// A storage READ failure here is surfaced by getNouns/getVerbs as a named
|
// A storage READ failure here is surfaced by getNouns/getVerbs as a named
|
||||||
// BrainyError — it means we could not even read the store to decide whether
|
// BrainyError — it means we could not even read the store to decide whether
|
||||||
|
|
@ -13083,6 +13176,25 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Re-stamp the 8.0 ⇄ native-provider version-handshake marker
|
||||||
|
* (`_system/brain-format.json`) to this build's {@link CURRENT_DATA_FORMAT} /
|
||||||
|
* {@link EXPECTED_INDEX_EPOCH} — but ONLY when the on-disk marker was absent
|
||||||
|
* or carried a drifted epoch ({@link _indexEpochStale}). Called at the
|
||||||
|
* verified-completion points of {@link rebuildIndexesIfNeeded} (after the
|
||||||
|
* derived-index rebuild, or on the empty-storage fast path), so the marker is
|
||||||
|
* never advanced ahead of the indexes it certifies — a crash before this
|
||||||
|
* point re-triggers the idempotent rebuild on the next open. Readers never
|
||||||
|
* write, so this is a no-op in reader mode.
|
||||||
|
*/
|
||||||
|
private async stampBrainFormatIfNeeded(): Promise<void> {
|
||||||
|
if (this.isReadOnly) return
|
||||||
|
if (!this._indexEpochStale) return
|
||||||
|
await writeBrainFormat(this.storage)
|
||||||
|
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
|
||||||
|
this._indexEpochStale = false
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check health of metadata indexes
|
* Check health of metadata indexes
|
||||||
*
|
*
|
||||||
|
|
|
||||||
136
src/storage/brainFormat.ts
Normal file
136
src/storage/brainFormat.ts
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
/**
|
||||||
|
* @module storage/brainFormat
|
||||||
|
* @description The 7.x → 8.0 version-handshake marker: the small persisted
|
||||||
|
* artifact at `_system/brain-format.json` that lets Brainy and a native
|
||||||
|
* metadata/index provider agree, at open time, on the on-disk format of the
|
||||||
|
* brain AND of its derived indexes.
|
||||||
|
*
|
||||||
|
* Two fields, two owners:
|
||||||
|
*
|
||||||
|
* - `dataFormat` — **Brainy-owned**. The data-layer version string (the
|
||||||
|
* canonical entity / relationship / generation-record layout). Brainy bumps
|
||||||
|
* it when that data layout changes; a native provider reads it only to
|
||||||
|
* confirm the major line it is binding against (e.g. "this is an 8.0 brain").
|
||||||
|
* - `indexEpoch` — **SHARED**. A monotonic integer that Brainy and the native
|
||||||
|
* provider bump TOGETHER, in lockstep, on any coordinated release where the
|
||||||
|
* on-disk format of ANY derived index (the HNSW vectors, the metadata
|
||||||
|
* postings, or the graph adjacency) changes. It is the single switch that
|
||||||
|
* declares "every derived index written by an older build is stale and must
|
||||||
|
* be rebuilt from the canonical records."
|
||||||
|
*
|
||||||
|
* Open-time handshake. On open, Brainy compares the on-disk `indexEpoch`
|
||||||
|
* against the compiled {@link EXPECTED_INDEX_EPOCH}. A mismatch — OR an absent
|
||||||
|
* marker (a pre-handshake brain, or a brand-new store) — means the derived
|
||||||
|
* indexes on disk predate this build, so Brainy rebuilds them from the
|
||||||
|
* canonical records and only THEN re-stamps the marker. The provider reads the
|
||||||
|
* same surface synchronously through `brain.formatInfo()` at its own provider
|
||||||
|
* init() to confirm "running data-format X, index epoch N" before binding its
|
||||||
|
* native readers.
|
||||||
|
*
|
||||||
|
* Lockstep contract. NEVER bump {@link EXPECTED_INDEX_EPOCH} unilaterally. It
|
||||||
|
* advances only on a coordinated release whose on-disk derived-index format
|
||||||
|
* actually changed, and both projects ship the SAME new value in the same
|
||||||
|
* release — so a brain written by either side is recognised as current by the
|
||||||
|
* other, and a brain written by an older build of either side is recognised as
|
||||||
|
* stale and rebuilt. The constant living here makes this module the single
|
||||||
|
* source of truth both sides reference.
|
||||||
|
*
|
||||||
|
* Non-destructive stamping. The marker is the LAST thing written, AFTER the
|
||||||
|
* rebuild has verified. A crash between the rebuild and the stamp leaves the
|
||||||
|
* old / absent marker on disk, so the next open re-detects the drift and
|
||||||
|
* re-runs the (idempotent) rebuild — the marker is never advanced ahead of the
|
||||||
|
* indexes it certifies. This mirrors the generational record layer's
|
||||||
|
* "build-new → verify → atomic-rename" commit discipline
|
||||||
|
* (`src/db/generationStore.ts`).
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The narrow storage surface the marker helpers need — the
|
||||||
|
* raw-object read/write primitives every `BaseStorage` adapter implements (the
|
||||||
|
* same surface the generational record layer uses for `_system/` artifacts).
|
||||||
|
* Declared structurally so this module carries no runtime dependency on the
|
||||||
|
* adapter class.
|
||||||
|
*/
|
||||||
|
export interface BrainFormatStorage {
|
||||||
|
/** Read a raw object at a storage-root-relative path (`null` if absent). */
|
||||||
|
readRawObject(path: string): Promise<unknown | null>
|
||||||
|
/** Write a raw object at a storage-root-relative path (atomic tmp+rename on disk). */
|
||||||
|
writeRawObject(path: string, data: unknown): Promise<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Storage-root-relative path of the version-handshake marker. */
|
||||||
|
export const BRAIN_FORMAT_PATH = '_system/brain-format.json'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The compiled derived-index format epoch this build expects on
|
||||||
|
* disk. SHARED and lockstep-bumped with the native provider: advance it (in
|
||||||
|
* BOTH projects, to the same value, in a coordinated release) on any change to
|
||||||
|
* the on-disk format of any derived index — never on one side alone. Start = 1
|
||||||
|
* (the 8.0 GA baseline). An on-disk `indexEpoch` that differs from this — or an
|
||||||
|
* absent marker — triggers a full derived-index rebuild on open.
|
||||||
|
*/
|
||||||
|
export const EXPECTED_INDEX_EPOCH = 1
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The data-layer format string this build writes and runs as.
|
||||||
|
* Brainy-owned; bumped when the canonical data layout changes. Start = '8.0'.
|
||||||
|
*/
|
||||||
|
export const CURRENT_DATA_FORMAT = '8.0'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The persisted shape of `_system/brain-format.json` — the
|
||||||
|
* value `brain.formatInfo()` returns for the running brain, and the value
|
||||||
|
* read back from disk to drive the epoch-drift rebuild trigger.
|
||||||
|
*/
|
||||||
|
export interface BrainFormat {
|
||||||
|
/** Brainy-owned data-layer version string (e.g. `'8.0'`). */
|
||||||
|
dataFormat: string
|
||||||
|
/** Shared, lockstep-bumped derived-index format epoch (e.g. `1`). */
|
||||||
|
indexEpoch: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Read the on-disk version-handshake marker, or `null` when it is
|
||||||
|
* absent (a pre-handshake brain, or a brand-new store). A malformed marker
|
||||||
|
* (not an object, missing either field, or a non-finite epoch) is also treated
|
||||||
|
* as `null` — a corrupt marker forces a safe rebuild rather than trusting a bad
|
||||||
|
* epoch.
|
||||||
|
* @param storage - The brain's storage adapter (raw-object surface).
|
||||||
|
* @returns The parsed {@link BrainFormat}, or `null`.
|
||||||
|
* @example
|
||||||
|
* const onDisk = await readBrainFormat(brain.storage)
|
||||||
|
* const stale = onDisk === null || onDisk.indexEpoch !== EXPECTED_INDEX_EPOCH
|
||||||
|
*/
|
||||||
|
export async function readBrainFormat(
|
||||||
|
storage: Pick<BrainFormatStorage, 'readRawObject'>
|
||||||
|
): Promise<BrainFormat | null> {
|
||||||
|
const raw = (await storage.readRawObject(BRAIN_FORMAT_PATH)) as Partial<BrainFormat> | null
|
||||||
|
if (raw === null || typeof raw !== 'object') return null
|
||||||
|
if (
|
||||||
|
typeof raw.dataFormat !== 'string' ||
|
||||||
|
typeof raw.indexEpoch !== 'number' ||
|
||||||
|
!Number.isFinite(raw.indexEpoch)
|
||||||
|
) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
return { dataFormat: raw.dataFormat, indexEpoch: raw.indexEpoch }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Stamp this build's {@link CURRENT_DATA_FORMAT} /
|
||||||
|
* {@link EXPECTED_INDEX_EPOCH} to `_system/brain-format.json` (an atomic
|
||||||
|
* tmp+rename on the filesystem adapter). MUST be called only AFTER the
|
||||||
|
* derived-index rebuild has verified: the marker certifies the indexes on
|
||||||
|
* disk, so advancing it ahead of them would suppress the rebuild a future open
|
||||||
|
* needs (the non-destructive contract — see the module docs).
|
||||||
|
* @param storage - The brain's storage adapter (raw-object surface).
|
||||||
|
*/
|
||||||
|
export async function writeBrainFormat(
|
||||||
|
storage: Pick<BrainFormatStorage, 'writeRawObject'>
|
||||||
|
): Promise<void> {
|
||||||
|
const marker: BrainFormat = {
|
||||||
|
dataFormat: CURRENT_DATA_FORMAT,
|
||||||
|
indexEpoch: EXPECTED_INDEX_EPOCH
|
||||||
|
}
|
||||||
|
await storage.writeRawObject(BRAIN_FORMAT_PATH, marker)
|
||||||
|
}
|
||||||
243
tests/unit/brainy/brain-format-handshake.test.ts
Normal file
243
tests/unit/brainy/brain-format-handshake.test.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
||||||
|
/**
|
||||||
|
* @module brain-format-handshake.test
|
||||||
|
* @description Tests for the 7.x → 8.0 version-handshake marker (GA #30): the
|
||||||
|
* `_system/brain-format.json` surface cor reads to drive whole-brain
|
||||||
|
* auto-upgrade, and Brainy's own JS-index rebuild-on-format-drift trigger.
|
||||||
|
*
|
||||||
|
* The contract (converged with the cor team — see `src/storage/brainFormat.ts`):
|
||||||
|
*
|
||||||
|
* - `formatInfo()` is a SYNC accessor returning the running build's
|
||||||
|
* `{ dataFormat, indexEpoch }` (the compiled constants).
|
||||||
|
* - On open, an on-disk `indexEpoch` that differs from `EXPECTED_INDEX_EPOCH`,
|
||||||
|
* or an absent marker, means the derived JS indexes are stale and must be
|
||||||
|
* rebuilt from the canonical records, then the marker is re-stamped — but
|
||||||
|
* only AFTER the rebuild verifies (non-destructive: a crash mid-rebuild
|
||||||
|
* leaves the old / absent marker, so the next open idempotently re-rebuilds).
|
||||||
|
*
|
||||||
|
* Persistence note: the derived JS indexes are in-memory and cold-load via
|
||||||
|
* `rebuild()` on every reopen, so "did a rebuild happen" is not, on its own, a
|
||||||
|
* signal that the EPOCH triggered it. The epoch-force (rebuild past the warm
|
||||||
|
* `size()>0` fast path) is therefore proven by a focused white-box test; the
|
||||||
|
* reopen tests assert the observable handshake effects — the marker content,
|
||||||
|
* whether it was re-stamped, and the rebuild-before-stamp ordering.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { describe, it, expect, afterEach, vi } from 'vitest'
|
||||||
|
import fs from 'node:fs'
|
||||||
|
import os from 'node:os'
|
||||||
|
import path from 'node:path'
|
||||||
|
import { Brainy } from '../../../src/index.js'
|
||||||
|
import { NounType } from '../../../src/types/graphTypes.js'
|
||||||
|
import { BaseStorage } from '../../../src/storage/baseStorage.js'
|
||||||
|
import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
|
||||||
|
import { MetadataIndexManager } from '../../../src/utils/metadataIndex.js'
|
||||||
|
import {
|
||||||
|
BRAIN_FORMAT_PATH,
|
||||||
|
CURRENT_DATA_FORMAT,
|
||||||
|
EXPECTED_INDEX_EPOCH
|
||||||
|
} from '../../../src/storage/brainFormat.js'
|
||||||
|
|
||||||
|
const CURRENT_MARKER = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
|
||||||
|
|
||||||
|
const tempDirs: string[] = []
|
||||||
|
const brains: Brainy[] = []
|
||||||
|
|
||||||
|
function makeTempDir(): string {
|
||||||
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-format-handshake-'))
|
||||||
|
tempDirs.push(dir)
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Open (and track) a filesystem brain rooted at `dir`. */
|
||||||
|
async function openFsBrain(dir: string): Promise<Brainy> {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await brain.init()
|
||||||
|
brains.push(brain)
|
||||||
|
return brain
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Read the on-disk marker through a fresh raw adapter (after the brain closed). */
|
||||||
|
async function readMarkerFromDisk(dir: string): Promise<unknown> {
|
||||||
|
const adapter = new FileSystemStorage(dir)
|
||||||
|
await adapter.init()
|
||||||
|
return adapter.readRawObject(BRAIN_FORMAT_PATH)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Overwrite (or delete, when `marker` is null) the on-disk marker between reopens. */
|
||||||
|
async function seedMarkerOnDisk(dir: string, marker: unknown | null): Promise<void> {
|
||||||
|
const adapter = new FileSystemStorage(dir)
|
||||||
|
await adapter.init()
|
||||||
|
if (marker === null) {
|
||||||
|
await adapter.deleteRawObject(BRAIN_FORMAT_PATH)
|
||||||
|
} else {
|
||||||
|
await adapter.writeRawObject(BRAIN_FORMAT_PATH, marker)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
for (const brain of brains.splice(0)) {
|
||||||
|
try {
|
||||||
|
await brain.close()
|
||||||
|
} catch {
|
||||||
|
// already closed by the test
|
||||||
|
}
|
||||||
|
}
|
||||||
|
vi.restoreAllMocks()
|
||||||
|
for (const dir of tempDirs.splice(0)) {
|
||||||
|
await fs.promises.rm(dir, { recursive: true, force: true })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('8.0 ⇄ cor version-handshake marker (_system/brain-format.json)', () => {
|
||||||
|
// (a) Fresh brain → formatInfo() is the current format; the marker is written.
|
||||||
|
it('(a) fresh brain: formatInfo() reports the current format and writes the marker', async () => {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
brains.push(brain)
|
||||||
|
|
||||||
|
expect(brain.formatInfo()).toEqual(CURRENT_MARKER)
|
||||||
|
|
||||||
|
const onDisk = await (brain as unknown as {
|
||||||
|
storage: { readRawObject(p: string): Promise<unknown> }
|
||||||
|
}).storage.readRawObject(BRAIN_FORMAT_PATH)
|
||||||
|
expect(onDisk).toEqual(CURRENT_MARKER)
|
||||||
|
})
|
||||||
|
|
||||||
|
// (b) Reopen a current-epoch brain → no re-stamp; formatInfo() consistent.
|
||||||
|
it('(b) reopen at the current epoch: no re-stamp, formatInfo() stays consistent', async () => {
|
||||||
|
const dir = makeTempDir()
|
||||||
|
const first = await openFsBrain(dir)
|
||||||
|
await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } })
|
||||||
|
await first.close()
|
||||||
|
brains.splice(brains.indexOf(first), 1)
|
||||||
|
|
||||||
|
// The on-disk marker is present and current; reopening must NOT rewrite it.
|
||||||
|
const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject')
|
||||||
|
const second = await openFsBrain(dir)
|
||||||
|
|
||||||
|
const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH)
|
||||||
|
expect(stampWrites.length).toBe(0)
|
||||||
|
expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false)
|
||||||
|
expect(second.formatInfo()).toEqual(CURRENT_MARKER)
|
||||||
|
})
|
||||||
|
|
||||||
|
// (c) Stale on-disk epoch → derived-index rebuild, THEN re-stamp (rebuild before stamp).
|
||||||
|
it('(c) stale on-disk epoch: rebuilds the derived indexes, then re-stamps (rebuild BEFORE stamp)', async () => {
|
||||||
|
const dir = makeTempDir()
|
||||||
|
const first = await openFsBrain(dir)
|
||||||
|
await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } })
|
||||||
|
await first.close()
|
||||||
|
brains.splice(brains.indexOf(first), 1)
|
||||||
|
|
||||||
|
// Simulate a brain written by an OLDER build: drift the epoch back to 0.
|
||||||
|
await seedMarkerOnDisk(dir, { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH - 1 })
|
||||||
|
|
||||||
|
// Spy AFTER the seed write so only the reopen's calls are observed. Both
|
||||||
|
// spies call through; vitest's `invocationCallOrder` is a single global
|
||||||
|
// counter, so cross-spy ordering is comparable.
|
||||||
|
const rebuildSpy = vi.spyOn(MetadataIndexManager.prototype, 'rebuild')
|
||||||
|
const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject')
|
||||||
|
|
||||||
|
const second = await openFsBrain(dir)
|
||||||
|
|
||||||
|
// The marker was re-stamped exactly once, to the EXPECTED epoch.
|
||||||
|
const stampOrders = writeSpy.mock.calls
|
||||||
|
.map((call, i) => ({ path: call[0] as string, order: writeSpy.mock.invocationCallOrder[i] }))
|
||||||
|
.filter((c) => c.path === BRAIN_FORMAT_PATH)
|
||||||
|
expect(stampOrders.length).toBe(1)
|
||||||
|
expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false)
|
||||||
|
|
||||||
|
// The derived index rebuilt, and the rebuild happened BEFORE the stamp
|
||||||
|
// (non-destructive ordering: the marker only advances once the rebuild that
|
||||||
|
// it certifies has run).
|
||||||
|
expect(rebuildSpy).toHaveBeenCalled()
|
||||||
|
const firstRebuildOrder = rebuildSpy.mock.invocationCallOrder[0]
|
||||||
|
expect(firstRebuildOrder).toBeLessThan(stampOrders[0].order)
|
||||||
|
|
||||||
|
await second.close()
|
||||||
|
brains.splice(brains.indexOf(second), 1)
|
||||||
|
|
||||||
|
// And the new marker is durable on disk at the EXPECTED epoch.
|
||||||
|
expect(await readMarkerFromDisk(dir)).toEqual(CURRENT_MARKER)
|
||||||
|
})
|
||||||
|
|
||||||
|
// (d) No marker (pre-handshake brain) → treated as stale → rebuild + stamp.
|
||||||
|
it('(d) no marker (pre-handshake brain): treated as stale → rebuilds and stamps the marker', async () => {
|
||||||
|
const dir = makeTempDir()
|
||||||
|
const first = await openFsBrain(dir)
|
||||||
|
await first.add({ data: 'x', type: NounType.Document, metadata: { k: 1 } })
|
||||||
|
await first.close()
|
||||||
|
brains.splice(brains.indexOf(first), 1)
|
||||||
|
|
||||||
|
// A brain written before the handshake existed has NO marker on disk.
|
||||||
|
await seedMarkerOnDisk(dir, null)
|
||||||
|
expect(await readMarkerFromDisk(dir)).toBeNull()
|
||||||
|
|
||||||
|
const rebuildSpy = vi.spyOn(MetadataIndexManager.prototype, 'rebuild')
|
||||||
|
const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject')
|
||||||
|
|
||||||
|
const second = await openFsBrain(dir)
|
||||||
|
|
||||||
|
expect(rebuildSpy).toHaveBeenCalled()
|
||||||
|
const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH)
|
||||||
|
expect(stampWrites.length).toBe(1)
|
||||||
|
expect((second as unknown as { _indexEpochStale: boolean })._indexEpochStale).toBe(false)
|
||||||
|
|
||||||
|
await second.close()
|
||||||
|
brains.splice(brains.indexOf(second), 1)
|
||||||
|
expect(await readMarkerFromDisk(dir)).toEqual(CURRENT_MARKER)
|
||||||
|
})
|
||||||
|
|
||||||
|
// The epoch-drift trigger itself: a format change forces a rebuild past the
|
||||||
|
// warm-index fast path (the gap this feature closes); a current epoch does not.
|
||||||
|
it('epoch drift forces a rebuild of all three indexes past the warm fast path; current epoch does not', async () => {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
brains.push(brain)
|
||||||
|
|
||||||
|
// Warm the indexes (size() > 0) so the fast-path early return is live.
|
||||||
|
await brain.add({ data: 'a', type: NounType.Document, metadata: { k: 1 } })
|
||||||
|
await brain.add({ data: 'b', type: NounType.Document, metadata: { k: 2 } })
|
||||||
|
const internals = brain as unknown as {
|
||||||
|
index: { size(): number; rebuild(...a: unknown[]): Promise<unknown> }
|
||||||
|
metadataIndex: { rebuild(...a: unknown[]): Promise<unknown> }
|
||||||
|
graphIndex: { rebuild(...a: unknown[]): Promise<unknown> }
|
||||||
|
_indexEpochStale: boolean
|
||||||
|
rebuildIndexesIfNeeded(force?: boolean): Promise<void>
|
||||||
|
}
|
||||||
|
expect(internals.index.size()).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined)
|
||||||
|
const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined)
|
||||||
|
const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined)
|
||||||
|
|
||||||
|
// Current epoch: the warm fast path skips every rebuild.
|
||||||
|
internals._indexEpochStale = false
|
||||||
|
await internals.rebuildIndexesIfNeeded()
|
||||||
|
expect(idxSpy).not.toHaveBeenCalled()
|
||||||
|
expect(miSpy).not.toHaveBeenCalled()
|
||||||
|
expect(giSpy).not.toHaveBeenCalled()
|
||||||
|
|
||||||
|
// Drifted epoch: all three rebuild even though the indexes are non-empty.
|
||||||
|
internals._indexEpochStale = true
|
||||||
|
await internals.rebuildIndexesIfNeeded()
|
||||||
|
expect(idxSpy).toHaveBeenCalledTimes(1)
|
||||||
|
expect(miSpy).toHaveBeenCalledTimes(1)
|
||||||
|
expect(giSpy).toHaveBeenCalledTimes(1)
|
||||||
|
// The stamp cleared the stale flag after the verified rebuild.
|
||||||
|
expect(internals._indexEpochStale).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
// (e) formatInfo() is synchronously available immediately after init (no await).
|
||||||
|
it('(e) formatInfo() is synchronously available immediately after init', async () => {
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
||||||
|
await brain.init()
|
||||||
|
brains.push(brain)
|
||||||
|
|
||||||
|
// No await on the call itself — it is a pure in-memory constant read.
|
||||||
|
const info = brain.formatInfo()
|
||||||
|
expect(info).toEqual(CURRENT_MARKER)
|
||||||
|
expect(typeof info.indexEpoch).toBe('number')
|
||||||
|
expect(typeof info.dataFormat).toBe('string')
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue