diff --git a/package.json b/package.json index 780dd13d..d2fb629f 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,10 @@ "import": "./dist/internals.js", "types": "./dist/internals.d.ts" }, + "./brain-format": { + "import": "./dist/storage/brainFormat.js", + "types": "./dist/storage/brainFormat.d.ts" + }, "./embeddings/wasm": { "import": "./dist/embeddings/wasm/index.js", "types": "./dist/embeddings/wasm/index.d.ts" diff --git a/src/brainy.ts b/src/brainy.ts index 82c05d21..899d4be2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -6420,6 +6420,30 @@ export class Brainy implements BrainyInterface { return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } } + /** + * @description Stamp `_system/brain-format.json` with this build's + * `{ dataFormat, indexEpoch }` ({@link CURRENT_DATA_FORMAT} / + * {@link EXPECTED_INDEX_EPOCH}). A native provider calls this once its + * background index migration (the no-freeze path, where brainy DEFERRED the + * rebuild because the provider's `isMigrating()` was true) has + * verified-and-swapped all derived indexes — brainy authors `dataFormat`, the + * provider never does. The marker is the single switch that declares the + * on-disk derived indexes current for this build's epoch, so it is advanced + * only after the indexes it certifies are in place. Unconditional and + * idempotent (writes the same two compiled constants every call); a no-op for + * read-only brains, which never author markers. + * @returns Resolves once the marker is durable on disk. + * @example + * // cor, after its build-new→verify→swap completes: + * await brain.stampBrainFormat() + */ + async stampBrainFormat(): Promise { + if (this.isReadOnly) return + await writeBrainFormat(this.storage) + this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + this._indexEpochStale = false + } + /** * @description Read the reified transaction log — one entry per committed * generation, carrying the committed generation, the commit timestamp, and @@ -12937,6 +12961,16 @@ export class Brainy implements BrainyInterface { return } + // No-freeze deference (rc.8): when the vector provider is running its own + // non-blocking background build-new→verify→swap migration, a first query must + // NOT trigger brainy's blocking force-rebuild — cor owns the index and serves + // correct reads from canonical until it verifies-and-swaps. Defer (no + // `lazyRebuildCompleted` latch) so the check re-runs each query: once the swap + // lands, `index.size() > 0` above ends the lazy path normally. + if (this.providerIsMigrating(this.index)) { + return + } + // Concurrency control: If rebuild is in progress, wait for it if (this.lazyRebuildInProgress && this.lazyRebuildPromise) { await this.lazyRebuildPromise @@ -13118,14 +13152,27 @@ export class Brainy implements BrainyInterface { // three from the canonical records (not just the empty ones below). const epochStale = this._indexEpochStale - const needsRebuild = - metadataStats.totalEntries === 0 || - hnswIndexSize === 0 || - needsGraphRebuild || - epochStale + // No-freeze deference (rc.8): when a native provider is running its OWN + // non-blocking background build-new→verify→swap migration, brainy must NOT + // rebuild that index — the provider owns it and serves correct reads from + // canonical until it verifies-and-swaps, then calls brain.stampBrainFormat(). + // Gated per-index, so a non-migrating sibling still rebuilds when it needs + // to; a migrating provider is skipped even under epoch-drift or size()===0. + const metadataMigrating = this.providerIsMigrating(this.metadataIndex) + const vectorMigrating = this.providerIsMigrating(this.index) + const graphMigrating = this.providerIsMigrating(this.graphIndex) + const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating + + const shouldRebuildMetadata = + (metadataStats.totalEntries === 0 || epochStale) && !metadataMigrating + const shouldRebuildVector = (hnswIndexSize === 0 || epochStale) && !vectorMigrating + const shouldRebuildGraph = (needsGraphRebuild || epochStale) && !graphMigrating + + const needsRebuild = shouldRebuildMetadata || shouldRebuildVector || shouldRebuildGraph if (!needsRebuild && !force) { - // All indexes already populated, no rebuild needed + // All indexes already populated (or owned by a background migration), no + // rebuild needed. return } @@ -13160,20 +13207,21 @@ export class Brainy implements BrainyInterface { // graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints // and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s // ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()). - if (needsGraphRebuild || epochStale) { + if (shouldRebuildGraph) { await this.hydrateIdMapperForGraphRebuild() } - // Rebuild all 3 indexes in parallel for performance - // Indexes load their data from storage (no recomputation). An epoch drift - // (`epochStale`) rebuilds ALL three regardless of their current size — - // the on-disk format changed, so every derived index is rebuilt from the - // canonical records. + // Rebuild the needed indexes in parallel for performance. Indexes load + // their data from storage (no recomputation). An epoch drift (`epochStale`) + // rebuilds every NON-migrating index regardless of its current size — the + // on-disk format changed, so each is rebuilt from the canonical records. A + // provider running its own background migration is skipped here (it owns + // its index until it verifies-and-swaps). const rebuildStartTime = Date.now() await Promise.all([ - metadataStats.totalEntries === 0 || epochStale ? this.metadataIndex.rebuild() : Promise.resolve(), - hnswIndexSize === 0 || epochStale ? this.index.rebuild() : Promise.resolve(), - needsGraphRebuild || epochStale ? this.graphIndex.rebuild() : Promise.resolve() + shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(), + shouldRebuildVector ? this.index.rebuild() : Promise.resolve(), + shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve() ]) const rebuildDuration = Date.now() - rebuildStartTime @@ -13189,8 +13237,11 @@ export class Brainy implements BrainyInterface { } // Consistency verification: metadata index must match storage entity count. - // If mismatch, the rebuild missed entities — force a second attempt. - if (metadataCountAfter === 0 && totalCount > 0) { + // If mismatch, the rebuild missed entities — force a second attempt. Skipped + // when the metadata provider is mid-migration: a 0 count there reflects the + // background swap in progress (it serves from canonical), not a missed + // rebuild, so forcing a second rebuild would break deference. + if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) { console.error( `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + `Forcing second rebuild.` @@ -13206,7 +13257,15 @@ export class Brainy implements BrainyInterface { // old / absent marker, so the next open re-detects the drift and re-runs // the (idempotent) rebuild; the marker is never advanced ahead of the // indexes it certifies. A no-op when the epoch was already current. - await this.stampBrainFormatIfNeeded() + // + // EXCEPT while a provider is mid-migration (the no-freeze path): brainy + // deferred that index's rebuild, so it is NOT yet at this epoch. The marker + // (which certifies ALL derived indexes) must not advance ahead of the index + // cor is still building — cor calls brain.stampBrainFormat() once its + // background build-new→verify→swap has verified-and-swapped. + if (!anyMigrating) { + await this.stampBrainFormatIfNeeded() + } } catch (error) { // A storage READ failure here is surfaced by getNouns/getVerbs as a named @@ -13244,6 +13303,27 @@ export class Brainy implements BrainyInterface { this._indexEpochStale = false } + /** + * @description Whether an index provider is running its own non-blocking + * background migration (the no-freeze build-new→verify→swap path). A native + * provider's `init()` flips an OPTIONAL `isMigrating(): boolean` to true the + * moment it detects a large epoch-drift and keeps it true until it has + * verified-and-swapped its new derived index. While true, brainy DEFERS its + * own rebuild of that index and lets the provider own it (the provider serves + * correct reads from canonical meanwhile) — never a minutes-long blocking + * rebuild-on-open. Feature-detected: a provider that omits `isMigrating` + * (every JS index) is treated as not migrating, so behavior is unchanged. + * @param provider - A registered index provider (metadata / vector / graph). + * @returns `true` only when the provider exposes `isMigrating()` and it reports + * an in-flight migration. + */ + private providerIsMigrating(provider: unknown): boolean { + return ( + typeof (provider as { isMigrating?: () => boolean }).isMigrating === 'function' && + (provider as { isMigrating: () => boolean }).isMigrating() === true + ) + } + /** * Check health of metadata indexes * diff --git a/src/plugin.ts b/src/plugin.ts index 2f66a2bb..6da4a971 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -125,6 +125,17 @@ export interface MetadataIndexProvider { flush(): Promise rebuild(): Promise + /** + * @description OPTIONAL. A native provider returns true from the moment its + * `init()` detects a large epoch-drift until its background + * build-new→verify→swap has verified-and-swapped. While true, brainy SKIPS its + * own rebuild for this provider and lets the provider's non-blocking background + * migration own the index (the no-freeze path); the provider serves correct + * reads from canonical meanwhile. Mirrors {@link MetadataIndexProvider.init} + * (and `isReady?()` on the graph provider). + */ + isMigrating?(): boolean + addToIndex(id: string, entityOrMetadata: any, skipFlush?: boolean, deferWrites?: boolean): Promise removeFromIndex(id: string, metadata?: any): Promise @@ -276,6 +287,16 @@ export interface GraphIndexProvider { */ init?(): Promise + /** + * @description OPTIONAL. A native provider returns true from the moment its + * `init()` detects a large epoch-drift until its background + * build-new→verify→swap has verified-and-swapped. While true, brainy SKIPS its + * own rebuild for this provider and lets the provider's non-blocking background + * migration own the index (the no-freeze path); the provider serves correct + * reads from canonical meanwhile. Mirrors `isReady?()` / `init?()`. + */ + isMigrating?(): boolean + /** * @description Entity ints reachable from `id` (1 hop), deduped. * @param id - The entity's interned int (from the shared idMapper). @@ -880,6 +901,17 @@ export interface VectorIndexProvider { rebuild(options?: any): Promise flush(): Promise getPersistMode(): 'immediate' | 'deferred' + + /** + * @description OPTIONAL. A native provider returns true from the moment its + * `init()` detects a large epoch-drift until its background + * build-new→verify→swap has verified-and-swapped. While true, brainy SKIPS its + * own rebuild for this provider and lets the provider's non-blocking background + * migration own the index (the no-freeze path); the provider serves correct + * reads from canonical meanwhile. Mirrors `isReady?()` / `init?()` on the + * other index providers. + */ + isMigrating?(): boolean } diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts new file mode 100644 index 00000000..758751fa --- /dev/null +++ b/tests/unit/brainy/migration-deference.test.ts @@ -0,0 +1,244 @@ +/** + * @module tests/unit/brainy/migration-deference + * @description rc.8 no-freeze auto-upgrade hooks — the deference seam that lets a + * native provider run an ONLINE, background build-new→verify→swap index migration + * while brainy stays out of the way (never a minutes-long blocking + * rebuild-on-open / first-query on a large-brain epoch-drift upgrade). + * + * Three hooks, locked with the cor team: + * + * - Hook 1 (deference): an OPTIONAL sync `isMigrating(): boolean` on each index + * provider (metadata / vector / graph). While it returns true, brainy SKIPS its + * own rebuild of that index — both in {@link Brainy.rebuildIndexesIfNeeded} + * (even under epoch-drift or `size()===0`) and on the large-path first-query + * lazy force-rebuild. A NON-migrating sibling still rebuilds when it needs to. + * - Hook 2: the public `brain.stampBrainFormat()` the provider calls once its + * background migration has verified-and-swapped, authoring the shared + * `_system/brain-format.json` marker. + * - Hook 3: the marker module is re-exported at `@soulcraft/brainy/brain-format` + * so cor reads the SAME `EXPECTED_INDEX_EPOCH` / `CURRENT_DATA_FORMAT` constants + * (single source of truth, no duplicated value). + * + * These paths are never exercised by brainy CI (cor registers no provider there), + * so the deference gates and the public stamp are pinned here with a white-box + * test-double provider — the same pattern as the cold-graph / handshake tests. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig } from '../../helpers/test-factory.js' +import { BaseStorage } from '../../../src/storage/baseStorage.js' +import { + BRAIN_FORMAT_PATH, + CURRENT_DATA_FORMAT, + EXPECTED_INDEX_EPOCH +} from '../../../src/storage/brainFormat.js' + +const CURRENT_MARKER = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } + +/** The white-box surface this suite drives on a live brain instance. */ +interface BrainInternals { + index: { size(): number; rebuild(...a: unknown[]): Promise } + metadataIndex: { rebuild(...a: unknown[]): Promise } + graphIndex: { size(): number; rebuild(...a: unknown[]): Promise } + _indexEpochStale: boolean + lazyRebuildCompleted: boolean + rebuildIndexesIfNeeded(force?: boolean): Promise + ensureIndexesLoaded(): Promise + storage: { readRawObject(p: string): Promise } +} + +const brains: Brainy[] = [] + +/** Open (and track) a warmed memory brain holding `count` document entities. */ +async function makeWarmBrain(count = 2, extraConfig: Record = {}): Promise { + const brain = new Brainy(createTestConfig(extraConfig)) + await brain.init() + brains.push(brain) + for (let i = 0; i < count; i++) { + await brain.add({ data: `doc-${i}`, type: NounType.Document, metadata: { k: i } }) + } + return brain +} + +/** Cast a brain to its white-box internals. */ +function internalsOf(brain: Brainy): BrainInternals { + return brain as unknown as BrainInternals +} + +/** Force `provider.isMigrating()` to a fixed value (the native provider's deference flag). */ +function setMigrating(provider: object, value: boolean): void { + ;(provider as { isMigrating?: () => boolean }).isMigrating = () => value +} + +afterEach(async () => { + for (const brain of brains.splice(0)) { + try { + await brain.close() + } catch { + // already closed by the test + } + } + vi.restoreAllMocks() +}) + +describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / brain-format export)', () => { + // --- Hook 1: per-index deference in rebuildIndexesIfNeeded ---------------- + + it('metadata provider isMigrating(): its rebuild is skipped under epoch-drift; vector + graph siblings still rebuild', async () => { + const brain = await makeWarmBrain() + const internals = internalsOf(brain) + + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + setMigrating(internals.metadataIndex, true) + // Epoch-drift would normally force ALL three to rebuild past the warm fast path. + internals._indexEpochStale = true + + await internals.rebuildIndexesIfNeeded() + + // The migrating provider owns its index — brainy does NOT rebuild it. + expect(miSpy).toHaveBeenCalledTimes(0) + // Non-migrating siblings still rebuild under the drift. + expect(idxSpy).toHaveBeenCalledTimes(1) + expect(giSpy).toHaveBeenCalledTimes(1) + // The marker is NOT advanced while a migration is in flight — cor stamps it + // when its build-new→verify→swap completes, so the stale flag stays set. + expect(internals._indexEpochStale).toBe(true) + }) + + it('vector provider isMigrating(): its rebuild is skipped under epoch-drift; metadata + graph siblings still rebuild', async () => { + const brain = await makeWarmBrain() + const internals = internalsOf(brain) + + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + setMigrating(internals.index, true) + internals._indexEpochStale = true + + await internals.rebuildIndexesIfNeeded() + + expect(idxSpy).toHaveBeenCalledTimes(0) + expect(miSpy).toHaveBeenCalledTimes(1) + expect(giSpy).toHaveBeenCalledTimes(1) + expect(internals._indexEpochStale).toBe(true) + }) + + it('graph provider isMigrating(): its rebuild is skipped under epoch-drift; metadata + vector siblings still rebuild', async () => { + const brain = await makeWarmBrain() + const internals = internalsOf(brain) + + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + setMigrating(internals.graphIndex, true) + internals._indexEpochStale = true + + await internals.rebuildIndexesIfNeeded() + + expect(giSpy).toHaveBeenCalledTimes(0) + expect(miSpy).toHaveBeenCalledTimes(1) + expect(idxSpy).toHaveBeenCalledTimes(1) + expect(internals._indexEpochStale).toBe(true) + }) + + it('a migrating provider is skipped even when its size()===0; a non-migrating size()===0 sibling still rebuilds', async () => { + const brain = await makeWarmBrain() + const internals = internalsOf(brain) + + const miSpy = vi.spyOn(internals.metadataIndex, 'rebuild').mockResolvedValue(undefined) + const idxSpy = vi.spyOn(internals.index, 'rebuild').mockResolvedValue(undefined) + const giSpy = vi.spyOn(internals.graphIndex, 'rebuild').mockResolvedValue(undefined) + + // No epoch drift this time: the rebuild trigger is purely "index is empty". + internals._indexEpochStale = false + // Vector index reports empty AND is migrating → its background swap owns it. + vi.spyOn(internals.index, 'size').mockReturnValue(0) + setMigrating(internals.index, true) + // Graph index reports empty and is NOT migrating → brainy must rebuild it. + vi.spyOn(internals.graphIndex, 'size').mockReturnValue(0) + + await internals.rebuildIndexesIfNeeded() + + // size()===0 would normally force the vector rebuild — deference suppresses it. + expect(idxSpy).toHaveBeenCalledTimes(0) + // The empty, non-migrating graph sibling still rebuilds. + expect(giSpy).toHaveBeenCalledTimes(1) + // Metadata has entries and no drift → no rebuild needed. + expect(miSpy).toHaveBeenCalledTimes(0) + }) + + // --- Hook 1: large-path first-query lazy force-rebuild deference ---------- + + it('lazy first-query force-rebuild is SKIPPED when the vector provider isMigrating()', async () => { + // disableAutoRebuild routes first queries through ensureIndexesLoaded() (the + // large-brain lazy path that would otherwise force a blocking rebuild). + const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) + const internals = internalsOf(brain) + + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) + // Simulate a cold/empty live vector index (cor is mid-swap, serving canonical). + vi.spyOn(internals.index, 'size').mockReturnValue(0) + internals.lazyRebuildCompleted = false + setMigrating(internals.index, true) + + await internals.ensureIndexesLoaded() + + // A query during cor's background swap must not trigger brainy's blocking rebuild. + expect(rebuildSpy).toHaveBeenCalledTimes(0) + }) + + it('lazy first-query force-rebuild STILL fires when the vector provider is not migrating (control)', async () => { + const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) + const internals = internalsOf(brain) + + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) + vi.spyOn(internals.index, 'size').mockReturnValue(0) + internals.lazyRebuildCompleted = false + // No isMigrating → not deferring. + + await internals.ensureIndexesLoaded() + + // Without deference, the cold empty index drives the lazy force-rebuild. + expect(rebuildSpy).toHaveBeenCalledTimes(1) + expect(rebuildSpy).toHaveBeenCalledWith(true) + }) + + // --- Hook 2: public stampBrainFormat() ----------------------------------- + + it('brain.stampBrainFormat() writes _system/brain-format.json with the current {dataFormat, indexEpoch}', async () => { + const brain = new Brainy(createTestConfig()) + await brain.init() + brains.push(brain) + const internals = internalsOf(brain) + + // Spy AFTER init so only the stamp call's write is observed (init already + // stamped the fresh brain). + const writeSpy = vi.spyOn(BaseStorage.prototype, 'writeRawObject') + + await brain.stampBrainFormat() + + const stampWrites = writeSpy.mock.calls.filter(([p]) => p === BRAIN_FORMAT_PATH) + expect(stampWrites.length).toBe(1) + expect(stampWrites[0][1]).toEqual(CURRENT_MARKER) + + // And the marker is durable on disk at the current epoch. + const onDisk = await internals.storage.readRawObject(BRAIN_FORMAT_PATH) + expect(onDisk).toEqual(CURRENT_MARKER) + }) + + // --- Hook 3: marker module export ---------------------------------------- + + it('the brain-format marker module exports the compiled epoch + data-format constants', () => { + // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both + // sides share ONE source of truth — no duplicated constant to drift. + expect(EXPECTED_INDEX_EPOCH).toBe(1) + expect(CURRENT_DATA_FORMAT).toBe('8.0') + }) +})