feat(8.0): no-freeze auto-upgrade hooks — isMigrating() deference + stampBrainFormat() + brain-format export
Three hooks so a native provider can run a non-blocking, online, background build-new→verify→swap index migration on a large-brain upgrade while brainy stays out of the way — no minutes-long blocking rebuild-on-open/first-query. - isMigrating?(): boolean — OPTIONAL on MetadataIndexProvider / GraphIndexProvider / VectorIndexProvider (mirrors isReady?()/init?(), feature-detected). While a provider reports migrating, brainy SKIPS its rebuild for that index — per-index, so a non-migrating sibling still rebuilds; skipped even under epoch-drift or size()===0 — in both rebuildIndexesIfNeeded and the lazy first-query force path. The provider serves correct reads from canonical until it verifies-and-swaps. - brain.stampBrainFormat() — public; the provider calls it once its background swap verifies, so brainy authors dataFormat (the provider never does). brainy withholds its own marker stamp while any provider is migrating, so the shared epoch is never advanced ahead of a deferred index. - ./brain-format export — the marker module's EXPECTED_INDEX_EPOCH / CURRENT_DATA_FORMAT are importable so a native provider shares the constant (no duplicate = no lockstep-drift). 8-case test; unit 1743 green.
This commit is contained in:
parent
bd6faf7499
commit
b6b919890c
4 changed files with 378 additions and 18 deletions
116
src/brainy.ts
116
src/brainy.ts
|
|
@ -6420,6 +6420,30 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Stamp `_system/brain-format.json` with this build's
|
||||
* `{ dataFormat, indexEpoch }` ({@link CURRENT_DATA_FORMAT} /
|
||||
* {@link EXPECTED_INDEX_EPOCH}). A native provider calls this once its
|
||||
* background index migration (the no-freeze path, where brainy DEFERRED the
|
||||
* rebuild because the provider's `isMigrating()` was true) has
|
||||
* verified-and-swapped all derived indexes — brainy authors `dataFormat`, the
|
||||
* provider never does. The marker is the single switch that declares the
|
||||
* on-disk derived indexes current for this build's epoch, so it is advanced
|
||||
* only after the indexes it certifies are in place. Unconditional and
|
||||
* idempotent (writes the same two compiled constants every call); a no-op for
|
||||
* read-only brains, which never author markers.
|
||||
* @returns Resolves once the marker is durable on disk.
|
||||
* @example
|
||||
* // cor, after its build-new→verify→swap completes:
|
||||
* await brain.stampBrainFormat()
|
||||
*/
|
||||
async stampBrainFormat(): Promise<void> {
|
||||
if (this.isReadOnly) return
|
||||
await writeBrainFormat(this.storage)
|
||||
this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH }
|
||||
this._indexEpochStale = false
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the reified transaction log — one entry per committed
|
||||
* generation, carrying the committed generation, the commit timestamp, and
|
||||
|
|
@ -12937,6 +12961,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return
|
||||
}
|
||||
|
||||
// No-freeze deference (rc.8): when the vector provider is running its own
|
||||
// non-blocking background build-new→verify→swap migration, a first query must
|
||||
// NOT trigger brainy's blocking force-rebuild — cor owns the index and serves
|
||||
// correct reads from canonical until it verifies-and-swaps. Defer (no
|
||||
// `lazyRebuildCompleted` latch) so the check re-runs each query: once the swap
|
||||
// lands, `index.size() > 0` above ends the lazy path normally.
|
||||
if (this.providerIsMigrating(this.index)) {
|
||||
return
|
||||
}
|
||||
|
||||
// Concurrency control: If rebuild is in progress, wait for it
|
||||
if (this.lazyRebuildInProgress && this.lazyRebuildPromise) {
|
||||
await this.lazyRebuildPromise
|
||||
|
|
@ -13118,14 +13152,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// three from the canonical records (not just the empty ones below).
|
||||
const epochStale = this._indexEpochStale
|
||||
|
||||
const needsRebuild =
|
||||
metadataStats.totalEntries === 0 ||
|
||||
hnswIndexSize === 0 ||
|
||||
needsGraphRebuild ||
|
||||
epochStale
|
||||
// No-freeze deference (rc.8): when a native provider is running its OWN
|
||||
// non-blocking background build-new→verify→swap migration, brainy must NOT
|
||||
// rebuild that index — the provider owns it and serves correct reads from
|
||||
// canonical until it verifies-and-swaps, then calls brain.stampBrainFormat().
|
||||
// Gated per-index, so a non-migrating sibling still rebuilds when it needs
|
||||
// to; a migrating provider is skipped even under epoch-drift or size()===0.
|
||||
const metadataMigrating = this.providerIsMigrating(this.metadataIndex)
|
||||
const vectorMigrating = this.providerIsMigrating(this.index)
|
||||
const graphMigrating = this.providerIsMigrating(this.graphIndex)
|
||||
const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating
|
||||
|
||||
const shouldRebuildMetadata =
|
||||
(metadataStats.totalEntries === 0 || epochStale) && !metadataMigrating
|
||||
const shouldRebuildVector = (hnswIndexSize === 0 || epochStale) && !vectorMigrating
|
||||
const shouldRebuildGraph = (needsGraphRebuild || epochStale) && !graphMigrating
|
||||
|
||||
const needsRebuild = shouldRebuildMetadata || shouldRebuildVector || shouldRebuildGraph
|
||||
|
||||
if (!needsRebuild && !force) {
|
||||
// All indexes already populated, no rebuild needed
|
||||
// All indexes already populated (or owned by a background migration), no
|
||||
// rebuild needed.
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -13160,20 +13207,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints
|
||||
// and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s
|
||||
// ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()).
|
||||
if (needsGraphRebuild || epochStale) {
|
||||
if (shouldRebuildGraph) {
|
||||
await this.hydrateIdMapperForGraphRebuild()
|
||||
}
|
||||
|
||||
// Rebuild all 3 indexes in parallel for performance
|
||||
// Indexes load their data from storage (no recomputation). An epoch drift
|
||||
// (`epochStale`) rebuilds ALL three regardless of their current size —
|
||||
// the on-disk format changed, so every derived index is rebuilt from the
|
||||
// canonical records.
|
||||
// Rebuild the needed indexes in parallel for performance. Indexes load
|
||||
// their data from storage (no recomputation). An epoch drift (`epochStale`)
|
||||
// rebuilds every NON-migrating index regardless of its current size — the
|
||||
// on-disk format changed, so each is rebuilt from the canonical records. A
|
||||
// provider running its own background migration is skipped here (it owns
|
||||
// its index until it verifies-and-swaps).
|
||||
const rebuildStartTime = Date.now()
|
||||
await Promise.all([
|
||||
metadataStats.totalEntries === 0 || epochStale ? this.metadataIndex.rebuild() : Promise.resolve(),
|
||||
hnswIndexSize === 0 || epochStale ? this.index.rebuild() : Promise.resolve(),
|
||||
needsGraphRebuild || epochStale ? this.graphIndex.rebuild() : Promise.resolve()
|
||||
shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(),
|
||||
shouldRebuildVector ? this.index.rebuild() : Promise.resolve(),
|
||||
shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve()
|
||||
])
|
||||
|
||||
const rebuildDuration = Date.now() - rebuildStartTime
|
||||
|
|
@ -13189,8 +13237,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
// Consistency verification: metadata index must match storage entity count.
|
||||
// If mismatch, the rebuild missed entities — force a second attempt.
|
||||
if (metadataCountAfter === 0 && totalCount > 0) {
|
||||
// If mismatch, the rebuild missed entities — force a second attempt. Skipped
|
||||
// when the metadata provider is mid-migration: a 0 count there reflects the
|
||||
// background swap in progress (it serves from canonical), not a missed
|
||||
// rebuild, so forcing a second rebuild would break deference.
|
||||
if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) {
|
||||
console.error(
|
||||
`[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` +
|
||||
`Forcing second rebuild.`
|
||||
|
|
@ -13206,7 +13257,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// old / absent marker, so the next open re-detects the drift and re-runs
|
||||
// the (idempotent) rebuild; the marker is never advanced ahead of the
|
||||
// indexes it certifies. A no-op when the epoch was already current.
|
||||
await this.stampBrainFormatIfNeeded()
|
||||
//
|
||||
// EXCEPT while a provider is mid-migration (the no-freeze path): brainy
|
||||
// deferred that index's rebuild, so it is NOT yet at this epoch. The marker
|
||||
// (which certifies ALL derived indexes) must not advance ahead of the index
|
||||
// cor is still building — cor calls brain.stampBrainFormat() once its
|
||||
// background build-new→verify→swap has verified-and-swapped.
|
||||
if (!anyMigrating) {
|
||||
await this.stampBrainFormatIfNeeded()
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
// A storage READ failure here is surfaced by getNouns/getVerbs as a named
|
||||
|
|
@ -13244,6 +13303,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this._indexEpochStale = false
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Whether an index provider is running its own non-blocking
|
||||
* background migration (the no-freeze build-new→verify→swap path). A native
|
||||
* provider's `init()` flips an OPTIONAL `isMigrating(): boolean` to true the
|
||||
* moment it detects a large epoch-drift and keeps it true until it has
|
||||
* verified-and-swapped its new derived index. While true, brainy DEFERS its
|
||||
* own rebuild of that index and lets the provider own it (the provider serves
|
||||
* correct reads from canonical meanwhile) — never a minutes-long blocking
|
||||
* rebuild-on-open. Feature-detected: a provider that omits `isMigrating`
|
||||
* (every JS index) is treated as not migrating, so behavior is unchanged.
|
||||
* @param provider - A registered index provider (metadata / vector / graph).
|
||||
* @returns `true` only when the provider exposes `isMigrating()` and it reports
|
||||
* an in-flight migration.
|
||||
*/
|
||||
private providerIsMigrating(provider: unknown): boolean {
|
||||
return (
|
||||
typeof (provider as { isMigrating?: () => boolean }).isMigrating === 'function' &&
|
||||
(provider as { isMigrating: () => boolean }).isMigrating() === true
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check health of metadata indexes
|
||||
*
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue