From 8fc553b126fc0ef0dc0d2be29e71db4143b93f1a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 13:29:11 -0700 Subject: [PATCH 01/42] fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index A canonical row persisted with vector: [] (a system row, a deferred embed not yet landed, or any other legitimately-unvectored record) is a normal, enumerable row -- but rebuild()'s storage walk had no guard against it. storage.getVectorIndexData() derives its answer from the row's own record, so it returns non-null for any existing noun whether or not that noun was ever actually indexed -- rebuild() admitted such rows into the live graph with a length-0 vector. A vector-less node could become the entry point (or occupy any graph position); the next real insert then ran a distance calculation against it and blew up with a dimension mismatch. Fix at two layers in src/hnsw/hnswIndex.ts: - rebuild() now skips any row whose vector.length === 0 before it ever becomes a graph node (one summary count line, never per-row spam), and restores the pinned dimension from the first real vector it loads -- previously the pin stayed null across a restart, since addItem/updateItem are the only sites that set it and rebuild() never goes through either. - addItem/updateItem now refuse a length-0 vector with a typed EmptyVectorIndexError instead of ever pinning dimension to 0 or storing a vector-less node, so no future fill/rebuild/load path can poison the index silently. getVectorSafe's lazy-load "not found" check also missed that an empty array is truthy -- tightened to catch it. IndexOperations.ts's ReplaceInVectorIndexOperation rollback paths now skip re-adding an oldVector of length 0 (never a legal index member) instead of attempting an illegal empty re-insert on rollback. biography.test.ts's final ledger-exactness assertion assumed every noun the lane creates is vectored, including the VFS root counted in vfsBaselineNouns -- but the root is deliberately persisted unvectored. Corrected the expected formula to exclude it. Adds tests/integration/index-skips-unvectored.test.ts pinning: rebuild() indexes only vectored rows with the dimension pinned correctly; clear() then real adds never trip a dimension mismatch; addItem/updateItem refuse a length-0 vector; and a crash/repair cycle stays dimension-consistent. --- src/hnsw/hnswIndex.ts | 92 ++++++- src/transaction/operations/IndexOperations.ts | 22 +- .../index-skips-unvectored.test.ts | 253 ++++++++++++++++++ tests/lifecycle/biography.test.ts | 12 +- 4 files changed, 371 insertions(+), 8 deletions(-) create mode 100644 tests/integration/index-skips-unvectored.test.ts diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 77e4f84d..3caf23ba 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -64,6 +64,34 @@ export class HnswFlushError extends Error { } } +/** + * @description Thrown by {@link JsHnswVectorIndex.addItem} / {@link + * JsHnswVectorIndex.updateItem} when handed a length-0 vector. A length-0 + * vector is the sanctioned "unvectored" shape for a canonical noun record + * (class-J: a VFS-system row, a deferred embed not yet landed, or any other + * legitimately-vector-less row) — but it is NEVER a legal INDEX insert. The + * index itself has no concept of "unvectored"; deciding that a row is + * unvectored and therefore skippable is the FILL/REBUILD/LOAD consumer's job + * (see {@link JsHnswVectorIndex.rebuild}), done BEFORE ever calling addItem. + * A length-0 vector reaching this point is a caller bug: silently accepting + * it would pin `this.dimension = 0` on an empty index (poisoning every real + * insert thereafter with a dimension mismatch) or store a vector-less node + * that a distance calculation can never safely compare against. Loud errors, + * never quiet losses — this throws instead of either. + */ +export class EmptyVectorIndexError extends Error { + constructor(public readonly id: string, operation: 'addItem' | 'updateItem') { + super( + `${operation}(${id}): refusing to index a length-0 vector — a length-0 vector is the ` + + `sanctioned "unvectored" shape for a canonical row, but it is never a legal index ` + + `insert. Callers that fill/rebuild/load the index must skip vector.length === 0 rows ` + + `themselves (unvectored = nothing to index, not an error at that layer); reaching ` + + `here with one is a caller bug.` + ) + this.name = 'EmptyVectorIndexError' + } +} + /** * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native @@ -580,6 +608,15 @@ export class JsHnswVectorIndex implements VectorIndexProvider { throw new Error('Vector is undefined or null') } + // THE INDEX REFUSES A LENGTH-0 VECTOR (see EmptyVectorIndexError's JSDoc): + // an empty vector is the sanctioned "unvectored" shape at the canonical + // layer, never a legal index member. Refusing here — loudly, before the + // dimension pin below — means no future fill/rebuild/load path can ever + // poison `this.dimension` to 0 or park a vector-less node in the graph. + if (vector.length === 0) { + throw new EmptyVectorIndexError(id, 'addItem') + } + // Set dimension on first insert if (this.dimension === null) { this.dimension = vector.length @@ -954,6 +991,13 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return } + // Same refusal as addItem (see EmptyVectorIndexError's JSDoc) — an + // in-place relink must never rewrite an already-indexed node down to the + // unvectored shape or poison the pinned dimension. + if (vector.length === 0) { + throw new EmptyVectorIndexError(id, 'updateItem') + } + if (this.dimension === null) { this.dimension = vector.length } else if (vector.length !== this.dimension) { @@ -1555,7 +1599,15 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } const loaded = await this.storage.getNounVector(noun.id) - if (!loaded) { + // `loaded` is a length-0 array (not null/undefined) for a canonical row + // that is legitimately unvectored — `![]` is FALSE (an empty array is + // truthy), so the bare `!loaded` check below would silently accept it + // as "found" and hand a dimension-0 vector to a distance calculation. + // A node only reaches this lazy-load path because it is a MEMBER of + // the live index (rebuild() now refuses to admit unvectored rows — see + // its JSDoc), so an empty vector here is never legitimate: treat it + // exactly like "not found", loudly. + if (!loaded || loaded.length === 0) { throw new Error(`Vector not found for noun ${noun.id}`) } @@ -1765,9 +1817,42 @@ export class JsHnswVectorIndex implements VectorIndexProvider { totalCount = result.totalCount || result.items.length + // UNVECTORED ROWS ARE NOT AN INDEX MEMBER (the class-J law): a canonical + // noun whose vector leg is `[]` (a VFS-root-style system row, a + // deferred embed not yet landed, or a best-effort fallback for an + // unreadable vector leg) is a normal, enumerable, countable row — it + // is simply not indexed. `storage.getVectorIndexData()` derives its + // {level, connections} answer straight from the noun's OWN record, so + // it returns non-null for every existing noun regardless of whether + // that noun ever actually reached `addItem()` — it cannot be used to + // decide indexability. `nounData.vector.length === 0` is the one + // truthful signal (mirrors the `noun.vector.length > 0` guards in + // {@link getVectorSafe} / {@link getVectorSync}): skip here, counted + // once in a summary line, never per-row spam. + let skippedUnvectored = 0 + // Process all nouns at once for (const nounData of result.items) { try { + if (!nounData.vector || nounData.vector.length === 0) { + skippedUnvectored++ + continue + } + + // Restore the pinned dimension from the first real vector this + // rebuild loads. `addItem`/`updateItem` only pin `this.dimension` + // on a LIVE insert — a fresh rebuild from storage never goes + // through either, so without this the pin stays `null` across a + // restart. A `null` pin means the very next insert (correct OR + // wrong length) silently BECOMES the new pin instead of being + // checked against the store's real dimension — the wrong-length + // case then fails much later and less clearly, inside a distance + // calculation against an already-loaded node, instead of here, + // immediately, with a named expected-vs-got mismatch. + if (this.dimension === null) { + this.dimension = nounData.vector.length + } + // Load HNSW graph data for this entity const hnswData = await this.storage.getVectorIndexData(nounData.id) @@ -1815,7 +1900,10 @@ export class JsHnswVectorIndex implements VectorIndexProvider { options.onProgress(loadedCount, totalCount) } - prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`) + prodLog.info( + `HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})` + + (skippedUnvectored > 0 ? ` — ${skippedUnvectored.toLocaleString()} unvectored row(s) skipped` : '') + ) } // Step 5: CRITICAL - Recover entry point if missing) diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index cfee5074..1bbbca88 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -324,8 +324,17 @@ export class ReplaceInVectorIndexOperation implements Operation { return async () => { // Restore the declared before-state in place (see class JSDoc for - // the item-did-not-exist posture). - await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) + // the item-did-not-exist posture). A length-0 oldVector means the row + // was never actually indexed before this op ran (a length-0 vector is + // never a legal index member — see EmptyVectorIndexError) — there is + // no in-place "restore to empty" for the provider to perform, so + // rollback removes the row instead, leaving the same "not indexed" + // state the row was in before execute(). + if (this.oldVector.length > 0) { + await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) + } else { + await this.index.removeItem(this.id, generation) + } } } @@ -336,9 +345,14 @@ export class ReplaceInVectorIndexOperation implements Operation { return async () => { // updateItem-style restore via the same adjacent pair, back to the - // declared before-state. + // declared before-state. Same length-0 carve-out as the updateItem + // path above: an empty oldVector was never a legal index member, so + // rollback just leaves the row removed rather than attempting an + // illegal empty re-add. await this.index.removeItem(this.id, generation) - await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + if (this.oldVector.length > 0) { + await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + } } } } diff --git a/tests/integration/index-skips-unvectored.test.ts b/tests/integration/index-skips-unvectored.test.ts new file mode 100644 index 00000000..c65aaa01 --- /dev/null +++ b/tests/integration/index-skips-unvectored.test.ts @@ -0,0 +1,253 @@ +/** + * @module tests/integration/index-skips-unvectored + * @description THE UNVECTORED-ROW CURE — two integration tests + * (`tests/lifecycle/biography.test.ts`'s Ch4/5/6 chapter and + * `tests/integration/clear-persistence.test.ts`'s multi-cycle test) started + * failing after a canonical-storage change made a vector-less row (the + * class-J shape: `vector: []`, e.g. the VFS root, a deferred embed not yet + * landed, or any other legitimately-unvectored canonical record) VISIBLE to + * the enumeration walk `getNounsWithPagination()` for the first time — before + * that change such rows were simply invisible to the walk. `hnswIndex.ts`'s + * `rebuild()` never guarded against that shape: it inserted every row the + * walk yielded into the live in-memory index, including ones with a length-0 + * vector, because `storage.getVectorIndexData()` derives its {level, + * connections} answer straight from the noun's OWN record — it returns + * non-null for ANY existing noun, whether or not that noun was ever actually + * indexed via `addItem()`. A vector-less node admitted into the graph could + * become the entry point (or occupy any graph position), and the very next + * real-vectored `addItem()` then ran a distance calculation against it — + * `cosineDistance` throws "Vectors must have the same dimensions" the moment + * one operand is a length-0 array. + * + * THE FIX, at two layers (`src/hnsw/hnswIndex.ts`): + * (1) FILL/REBUILD/LOAD consumers treat `vector.length === 0` as "unvectored — + * nothing to index" and skip the row (normal, not an error; one summary + * count line, never per-row spam) — `rebuild()`'s loop now checks this + * BEFORE ever creating a graph node, so an unvectored row can never + * become an index member, entry point, or dimension-setter. + * (2) THE INDEX ITSELF refuses a length-0 vector in `addItem()` / + * `updateItem()` with a typed `EmptyVectorIndexError`, loudly, instead of + * ever pinning `dimension = 0` or storing a vector-less node — so no + * future fill/rebuild/load path can silently poison the index even if it + * forgets law (1). + * + * Four legs pinned here: + * (a) `rebuild()` over a store mixing real-vectored rows and `vector: []` + * rows indexes ONLY the vectored ones — size === vectored count, + * dimension pinned to the real (non-zero) length. + * (b) `clear()` then real adds afterward never trip a dimension mismatch — + * the exact `clear-persistence.test.ts` regression shape, reproduced + * directly against the index/storage seam this module owns. + * (c) `index.addItem({ id, vector: [] })` throws `EmptyVectorIndexError` + * (and `updateItem` does too, for an existing node). + * (d) crash -> repair: the crashed generation's entities survive, the ledger + * recounts honestly, and a fresh real-vectored add afterward never trips + * a dimension mismatch against a leftover vector-less phantom. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { EmptyVectorIndexError } from '../../src/hnsw/hnswIndex.js' +import { abandonAsCrashed, openBrain as openKillMatrixBrain, uid, vec } from '../helpers/durabilityKillMatrix.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) { + try { + fs.rmSync(d, { recursive: true, force: true }) + } catch { + /* best-effort cleanup */ + } + } +}) + +/** A filesystem-backed brain with explicit vectors (no embedder needed) and + * manual persistence — mirrors `durabilityKillMatrix.ts`'s `openBrain` so + * every write in this module is explicit and provably durable. */ +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + persistence: { policy: 'manual' } + }) +} + +describe('HNSW index skips unvectored rows', () => { + it('(a) rebuild() indexes only vectored rows: size === vectored count, dimension pinned to the real length', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() + + // 5 real-vectored rows. + const vectoredIds: string[] = [] + for (let i = 0; i < 5; i++) { + const id = uid(`vectored-${i}`) + await brain.add({ id, data: `real entity ${i}`, type: NounType.Document, vector: vec(i) }) + vectoredIds.push(id) + } + // 3 explicit unvectored rows — the class-J "vector: []" shape, a normal, + // enumerable, countable canonical row that must never reach the index. + const unvectoredIds: string[] = [] + for (let i = 0; i < 3; i++) { + const id = uid(`unvectored-${i}`) + await brain.add({ id, data: `unvectored entity ${i}`, type: NounType.Document, vector: [] }) + unvectoredIds.push(id) + } + await brain.flush() + + // The canonical ledger already agrees before any rebuild: nouns.all + // counts every row (8 + the VFS root); vectors.all counts only the real + // ones (5) — the VFS root and the 3 explicit unvectored rows are excluded. + const ledgerBeforeReopen = await brain.storage.getCanonicalCounts() + expect(ledgerBeforeReopen.vectors.all).toBe(5) + expect(ledgerBeforeReopen.nouns.all).toBe(9) // 5 vectored + 3 unvectored + 1 VFS root + + await brain.close() + + // Reopen: open()'s index build IS hnswIndex.rebuild() run fresh from + // storage — this is the exact path that used to admit unvectored rows. + brain = openBrain(dir) + await brain.init() + + const status = await brain.getIndexStatus() + expect(status.hnswIndex.size, 'the rebuilt index must contain ONLY the 5 real-vectored rows').toBe(5) + + // Dimension is pinned to the REAL embedded length (384 via `vec()`), not + // 0 — adding a wrong-length vector must be refused naming that real + // dimension, proving no vector-less row ever set it. + const realDimension = vec(0).length + let mismatchMessage: string | undefined + try { + await brain.index.addItem({ id: uid('dimension-probe'), vector: vec(0).slice(0, realDimension - 1) }) + expect.fail('expected a dimension mismatch error') + } catch (err) { + mismatchMessage = (err as Error).message + } + expect(mismatchMessage).toContain(`expected ${realDimension}`) + + // Every unvectored row is still a normal, enumerable, readable canonical + // record — class-J semantics survive the rebuild fix untouched. + for (const id of unvectoredIds) { + const entity = await brain.get(id, { includeVectors: true }) + expect(entity, `unvectored entity ${id} must remain readable`).not.toBeNull() + expect(entity.vector).toEqual([]) + } + // A correct-dimension add succeeds cleanly against the pinned dimension. + const freshId = uid('post-reopen-fresh') + await expect(brain.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(50) })).resolves.toBe( + freshId + ) + + await brain.close() + }) + + it('(b) clear() then real adds afterward never trip a dimension mismatch (the clear-persistence regression shape)', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() // the VFS root (vector: []) is the store's only row + + await brain.clear() + await brain.close() + + // Reopen over a store whose only surviving row is the recreated, + // unvectored VFS root — this is exactly the shape that used to poison + // the entry point / dimension in `clear-persistence.test.ts`. + brain = openBrain(dir) + await brain.init() + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(0) + + const id1 = uid('after-clear-1') + await expect(brain.add({ id: id1, data: 'after clear 1', type: NounType.Document, vector: vec(1) })).resolves.toBe( + id1 + ) + const id2 = uid('after-clear-2') + await expect(brain.add({ id: id2, data: 'after clear 2', type: NounType.Document, vector: vec(2) })).resolves.toBe( + id2 + ) + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(2) + + await brain.close() + }) + + it('(c) index.addItem/updateItem refuse a length-0 vector with EmptyVectorIndexError', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + await expect(brain.index.addItem({ id: uid('empty-add'), vector: [] })).rejects.toThrow(EmptyVectorIndexError) + + // updateItem on an EXISTING (real-vectored) node must refuse the same way. + const existingId = uid('existing-for-update') + await brain.add({ id: existingId, data: 'existing', type: NounType.Document, vector: vec(9) }) + await expect(brain.index.updateItem({ id: existingId, vector: [] })).rejects.toThrow(EmptyVectorIndexError) + + // The index was never disturbed by either refused call. + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(1) + + await brain.close() + }) + + it('(d) crash -> repair: the crashed generation survives, the ledger recounts honestly, and a fresh add afterward never trips a dimension mismatch', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-crash-')) + try { + let brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) + + // Baseline: real-vectored entities, durably flushed. + const baselineIds: string[] = [] + for (let i = 0; i < 5; i++) { + const id = uid(`baseline-${i}`) + await brain.add({ id, data: `baseline entity ${i}`, type: NounType.Document, vector: vec(i) }) + baselineIds.push(id) + } + await brain.flush() + + // Crash window: at-ack writes that are never flushed before the crash. + const crashedIds: string[] = [] + for (let i = 0; i < 4; i++) { + const id = uid(`crashed-${i}`) + await brain.add({ id, data: `crash-window entity ${i}`, type: NounType.Document, vector: vec(100 + i) }) + crashedIds.push(id) + } + await abandonAsCrashed(brain) + + // Reopen — logAuthority: 'adopt' replays the at-ack log for the crash window. + brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) + for (const id of [...baselineIds, ...crashedIds]) { + expect(await brain.get(id), `entity ${id} must survive the crash`).not.toBeNull() + } + + // Repair — must not disturb any entity, and must recount the ledger honestly. + const report = await brain.repairIndex() + expect(report.families.length).toBeGreaterThan(0) + for (const id of [...baselineIds, ...crashedIds]) { + expect(await brain.get(id), `entity ${id} must survive repair`).not.toBeNull() + } + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.vectors.all).toBe(baselineIds.length + crashedIds.length) + + // Second life: a fresh real-vectored add must never trip a dimension + // mismatch against a vector-less phantom left in the index — the exact + // mechanism `clear-persistence.test.ts` and the biography lane hit. + const secondLifeId = uid('second-life') + await expect( + brain.add({ id: secondLifeId, data: 'second life entity', type: NounType.Document, vector: vec(200) }) + ).resolves.toBe(secondLifeId) + expect(await brain.get(secondLifeId)).not.toBeNull() + + await brain.close() + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts index 8b274fce..274f0ef0 100644 --- a/tests/lifecycle/biography.test.ts +++ b/tests/lifecycle/biography.test.ts @@ -382,9 +382,17 @@ describe.sequential('lifecycle — the working store', () => { }, // Every noun this biography ever adds carries an explicit/computed // vector (the harness never defers an embed), so the vectored-noun - // scalar tracks nouns.all exactly. + // scalar tracks nouns.all exactly EXCEPT for the VFS root counted + // in `vfsBaselineNouns`: the root is deliberately persisted with + // `vector: []` (the sanctioned "unvectored" shape — see + // VirtualFileSystem.doInitializeRoot()'s zero-norm-avoidance + // comment) so it never pays the WASM engine's cold-compile cost and + // never crosses an engine boundary as a false attractor. It is the + // ONE hidden-tier record `vfsBaselineNouns` represents (see + // biographyHarness's module header), so it is excluded here even + // though it counts toward `nouns.all`. vectors: { - all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns + all: aliveEntities.length + model.vfsFileNouns }, suspect: false }) From 0de7665930ac23fb48cff0c0034f4fb475527727 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 13:53:09 -0700 Subject: [PATCH 02/42] fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine-pair seam law: a zero-norm vector never crosses an engine boundary. The index belt already refused to insert one, but the canonical write and the vectored-noun ledger still counted it, so a near-empty store whose only vectored row was zero-norm read "1 canonical vectored vs 0 indexed" and threw a not-ready error at open, and a store's own legacy zero-norm VFS root could trip the same gate before its VFS-init-time cure ever ran. - add()/update() (single and transact()) now normalize an explicit real all-zero vector to the unvectored [] shape before the dimension pin, the ledger flag, and the index ops ever see it (loud, one warn per write, canonical write still succeeds). - The legacy counts.json derivation walk (scanVectoredNounCount) excludes a persisted zero-norm row, matching the live ledger's definition. - A legacy zero-norm VFS root now migrates at open, before the vector-leg gate evaluates, via one O(1) fixed-path read (torn-tolerant — skips rather than aborting init on a torn root, letting the recovery walk heal it) — independent of whether a VirtualFileSystem is ever constructed this session. - update({ id, vector: [] }) (and the same op inside transact()) is now the sanctioned, idempotent unvector door: index removal, exactly-once ledger decrement, no re-embed, and it clears a pending deferred-embed marker rather than leaving it to re-vectorize the row later. The combination with deferEmbedding is a typed refusal. - JsHnswVectorIndex.rebuild() now skips a zero-norm/empty persisted vector when repopulating from canonical (the same belt the live add/replace paths already had), and health()'s index-parity check now compares HNSW size against the vectored-noun ledger rather than the raw metadata-entry count, since a store's VFS root is permanently unvectored by design. Co-Authored-By: Claude Fable 5 --- src/brainy.ts | 351 +++++++++++++-- src/hnsw/hnswIndex.ts | 29 +- src/storage/adapters/fileSystemStorage.ts | 30 +- src/utils/paramValidation.ts | 23 +- tests/integration/vfs-root-zero-norm.test.ts | 31 +- .../zero-norm-unvector-door.test.ts | 399 ++++++++++++++++++ 6 files changed, 810 insertions(+), 53 deletions(-) create mode 100644 tests/integration/zero-norm-unvector-door.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index bfac8c08..ed958a67 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -396,6 +396,15 @@ interface PlannedTransact { * marker outlives its write. */ markerRecords: FactMarkerRecord[] + /** + * Ids the batch's `{ op: 'update' }` unvector door (`vector: []`) needs to + * decrement on the vectored-noun ledger — consumed by `transact()` with a + * proper `await this.storage.noteVectorUnlanded?.(id)` per id, AFTER + * `commitTransaction` resolves (never for a rejected batch). Kept separate + * from `postCommit` (`Array<() => void>`, called synchronously, fire-and- + * forget) because the ledger hook is async and must be awaited. + */ + vectorUnlands: string[] } /** @@ -1504,6 +1513,15 @@ export class Brainy implements BrainyInterface { }).backfillBlobHistoryRefCountsIfNeeded() } + // LEG C (zero-norm/unvector-door law): migrate a legacy zero-norm VFS + // root BEFORE the vector-leg open gate below ever compares the + // canonical vectored-noun count against the vector index's size — see + // migrateLegacyZeroNormVfsRootIfNeeded's JSDoc for why this is a safe + // O(1) exception to "nothing at open may scale with brain size", and + // why it must run here rather than waiting on VirtualFileSystem's own + // (VFS-instance-gated) lazy migration. + await this.migrateLegacyZeroNormVfsRootIfNeeded() + // Rebuild indexes if needed for existing data. Runs to completion before // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true @@ -1518,8 +1536,9 @@ export class Brainy implements BrainyInterface { // cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph // index construction, the eager cold-load, id-resolver + connections- // codec wiring, crash-recovery index rebuild, the replay-gap check, - // legacy VFS blob adoption, blob-history backfill, and the - // rebuildIndexesIfNeeded() gate + migration check. + // legacy VFS blob adoption, blob-history backfill, the legacy + // zero-norm VFS root migration, and the rebuildIndexesIfNeeded() gate + // + migration check. markPhase('index-init-gate') // Register shutdown hooks for graceful count flushing (once globally) @@ -2912,10 +2931,34 @@ export class Brainy implements BrainyInterface { // vector shape, is structurally impossible). The background worker // embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed + let vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) + // THE ZERO-NORM LAW (canonical write side): a zero-norm vector is not a + // vector — it never crosses an engine boundary (the engine pair's seam + // law). This engine's own cosine distance treats an all-zero vector + // safely (a zero-norm operand always scores MAXIMUM distance — see + // isZeroNormVector's JSDoc), but a downstream engine serving squared- + // euclidean distance cannot tell it apart from a legitimate origin + // point — a false attractor that silently darkened 150+ rows in a + // production deployment. The index belt (AddToVectorIndexOperation) + // already refuses to INDEX a zero-norm vector, but until now the + // CANONICAL write still persisted it and the vectored-noun ledger + // counted it — so a near-empty store whose only vectored row was + // zero-norm read "canonical vectored > 0, index size 0" and threw a + // not-ready error at open. Normalize HERE, before the dimension pin, + // the vectored-ledger flag (`SaveNounMetadataOperation`'s `hasVector`), + // and the index ops below ever see it, so it persists as the sanctioned + // "unvectored" `[]` shape instead — the canonical write still succeeds. + if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { + prodLog.warn( + `[Brainy] add(): entity ${id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + vector = [] + } + // Ensure dimensions are set (a deferred-embed stub carries no dimension // information — the worker's real vector goes through the same guard). // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose @@ -3623,25 +3666,53 @@ export class Brainy implements BrainyInterface { // often the host writes. const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged + + // THE ZERO-NORM LAW (canonical write side) — see add()'s matching + // comment: an explicit REAL all-zero vector is not a vector. Normalize + // to the sanctioned "unvectored" `[]` shape BEFORE the dimension + // check, the unvector-door decision below, and the index ops ever see + // it — a local copy; `params.vector` itself is never mutated. + let explicitVector = params.vector + if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { + prodLog.warn( + `[Brainy] update(): entity ${params.id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + explicitVector = [] + } + + // THE SANCTIONED UNVECTOR DOOR: `explicitVector` at length 0 (an + // explicit `vector: []`, or a real all-zero vector just normalized + // above) is an instruction to remove the vector NOW — never "please + // embed". `validateUpdateParams` already refuses combining it with + // `deferEmbedding: true` (an empty array is truthy, so that guard + // fires unconditionally on any explicit `vector`). Idempotent on an + // already-unvectored row: the ledger decrement near the end of this + // method is gated on the PRIOR vector actually having been real. + const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 + // MT5 deferred re-embedding: the OLD vector keeps serving semantic // search — stale-but-present, never absent (the flicker law) — until // the background worker embeds the new data and swaps it atomically. const deferringEmbed = - params.deferEmbedding === true && hasNewData && !params.vector - if (params.vector) { - if (this.dimensions && params.vector.length !== this.dimensions) { + params.deferEmbedding === true && hasNewData && !explicitVector + if (explicitVector) { + // A length-0 explicit vector (the unvector door) carries no + // dimension information — exempt from the check, mirroring add()'s + // own `vector.length > 0` gate on the dimension pin. + if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` ) } - vector = params.vector + vector = explicitVector } else if (hasNewData && !deferringEmbed) { vector = await this.embed(params.data) } // A deferred data change does NOT reindex now (the vector is unchanged; // the worker's atomic swap carries the real reindex later). const needsReindexing = Boolean( - (hasNewData && !deferringEmbed) || params.type || params.vector + (hasNewData && !deferringEmbed) || params.type || explicitVector ) // Always update the noun with new metadata @@ -3735,6 +3806,22 @@ export class Brainy implements BrainyInterface { ? [this.enqueuePendingEmbed(params.id)] : undefined + // Leg D — the unvector door clears a PENDING deferred-embed marker: + // without this, the worker would later embed this row's current data + // and silently re-vector it, defeating the caller's explicit "remove + // the vector now" instruction. The clear rides THIS SAME commit fact + // (an `embed.landed` record with an empty vector — the recovery fold + // disarms a pending marker on ANY `embed.landed` for the id, + // regardless of the vector it carries), so a crash between the write + // and the in-memory clear below still recovers disarmed. Mutually + // exclusive with `embedMarkers` above: `deferringEmbed` requires an + // ABSENT `explicitVector`, so the two branches never both apply. + const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) + const commitRecords: FactMarkerRecord[] | undefined = + embedMarkers ?? (clearsPendingEmbed + ? [{ type: 'embed.landed', id: params.id, vector: [] }] + : undefined) + // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { @@ -3823,7 +3910,33 @@ export class Brainy implements BrainyInterface { } } ] - : undefined, embedMarkers) + : undefined, commitRecords) + + // Leg D continued — the in-memory pending-embed clear runs only AFTER + // the commit above actually succeeded (an aborted update must not + // disarm a marker whose durable `embed.landed` twin was never + // written). + if (clearsPendingEmbed) { + this.clearPendingEmbed(params.id) + prodLog.warn( + `[Brainy] update(): entity ${params.id} had a pending deferred embed — ` + + `the unvector door cleared it ('vector: []' is an explicit instruction, ` + + `never "please embed").` + ) + } + + // Leg D — vectored-ledger decrement for the sanctioned unvector door. + // update()'s own metadata write goes through UpdateNounMetadataOperation + // (isNew=false), so the saveNounMetadata(..., hasVector) seam never + // fires here — noteVectorUnlanded is the ONLY seam, the same + // sanctioned hook unvectorNounForRootMigration() uses. Gated on the + // PRIOR vector having actually been real (non-empty, non-zero-norm): + // an already-unvectored row's second call is a true no-op — no + // decrement, matching the ledger-exactness law (never double-count, + // never drift negative). + if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { + await this.storage.noteVectorUnlanded?.(params.id) + } // Aggregation hook (outside transaction — derived data). `existing` is // the full get() view — every reserved field top-level — and must be @@ -9179,6 +9292,15 @@ export class Brainy implements BrainyInterface { hook() } + // Leg D — vectored-ledger decrements for this batch's unvector-door + // updates (see planTxUpdate's matching comment), applied after the + // commit point and properly awaited (unlike `postCommit`'s synchronous + // fire-and-forget hooks) — each is the same sanctioned hook + // unvectorNounForRootMigration() uses. + for (const id of plan.vectorUnlands) { + await this.storage.noteVectorUnlanded?.(id) + } + // Change feed: the batch's events share its single committed generation. // A rejected batch throws at commitTransaction and never reaches here. this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) @@ -10422,7 +10544,8 @@ export class Brainy implements BrainyInterface { casUpdates: [], createdNouns: new Set(), changeEvents: [], - markerRecords: [] + markerRecords: [], + vectorUnlands: [] } for (const op of ops) { @@ -10544,9 +10667,22 @@ export class Brainy implements BrainyInterface { // marker-less committed row would be a silently missing vector, which is // the disallowed direction). The background worker embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed + let vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) + + // THE ZERO-NORM LAW — see the single-add() insert path's matching + // comment (a zero-norm vector is not a vector; never crosses an engine + // boundary). Normalized here BEFORE the dimension pin and the + // vectored-ledger `hasVector` flag below ever see it. + if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { + prodLog.warn( + `[Brainy] transact add: entity ${id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + vector = [] + } + // Gated on `vector.length > 0` — see the single-add() insert path's // matching comment: an explicit `vector: []` carries no dimension // information either, deferred or not. @@ -10717,17 +10853,62 @@ export class Brainy implements BrainyInterface { const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged let vector = existing.vector - if (params.vector) { - if (this.dimensions && params.vector.length !== this.dimensions) { + + // THE ZERO-NORM LAW + THE SANCTIONED UNVECTOR DOOR — transact() mirror + // of update()'s matching block: an explicit REAL all-zero vector + // normalizes to `[]` (never crosses an engine boundary), and an + // explicit `vector: []` (post-normalization) is the sanctioned unvector + // instruction, exempt from the dimension check. `validateUpdateParams` + // already refuses combining it with `deferEmbedding: true`. + let explicitVector = params.vector + if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { + prodLog.warn( + `[Brainy] transact update: entity ${params.id} was given an explicit all-zero ` + + `vector — a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + explicitVector = [] + } + const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 + + if (explicitVector) { + if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` ) } - vector = params.vector + vector = explicitVector } else if (hasNewData) { vector = await this.embed(params.data) } - const needsReindexing = Boolean(hasNewData || params.type || params.vector) + const needsReindexing = Boolean(hasNewData || params.type || explicitVector) + + // Leg D — the unvector door clears a PENDING deferred-embed marker (see + // update()'s matching comment for the full rationale): the durable + // clear (an `embed.landed` record, empty vector) rides the batch's ONE + // commit fact via `plan.markerRecords`; the in-memory clear is deferred + // to `plan.postCommit` so an aborted batch never disarms a marker whose + // durable twin was never written. + const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) + if (clearsPendingEmbed) { + plan.markerRecords.push({ type: 'embed.landed', id: params.id, vector: [] }) + plan.postCommit.push(() => { + this.clearPendingEmbed(params.id) + prodLog.warn( + `[Brainy] transact update: entity ${params.id} had a pending deferred embed — ` + + `the unvector door cleared it ('vector: []' is an explicit instruction, ` + + `never "please embed").` + ) + }) + } + + // Leg D — vectored-ledger decrement for the sanctioned unvector door, + // deferred to `plan.vectorUnlands` (consumed with a proper `await` in + // `transact()`, AFTER the commit succeeds — see its matching comment). + // Gated on the PRIOR vector having actually been real (non-empty, + // non-zero-norm): idempotent on an already-unvectored row. + if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { + plan.vectorUnlands.push(params.id) + } const newMetadata = params.merge !== false @@ -12431,21 +12612,49 @@ export class Brainy implements BrainyInterface { const metadataStats = await this.metadataIndex.getStats() const graphSize = await this.graphIndex.size() - // 1. Index size parity. HNSW must hold at least one node per indexed entity. - if (hnswSize === metadataStats.totalEntries) { + // 1. Index size parity. HNSW must hold one node per VECTORED noun — the + // vectored-noun ledger (`getCanonicalCounts().vectors.all`), NOT the raw + // metadata-entry count: every store's VFS root is PERMANENTLY unvectored + // (`vector: []` by design — a zero-norm/empty vector never crosses into + // the index, see AddToVectorIndexOperation/JsHnswVectorIndex.rebuild()'s + // matching belts), and a not-yet-landed deferred embed is unvectored + // too. Comparing against total entries counted the always-unvectored + // root as a permanent 1-node "drift" on every VFS-having store — a false + // warn on an otherwise perfectly healthy handoff. `vectors.all` is + // already the documented coverage denominator for exactly this + // comparison (see `CanonicalCounts.vectors`'s JSDoc). Falls back to the + // metadata-entry count when the ledger is unavailable or suspect (a + // storage adapter without the optional hook, or an unrecounted store) — + // never worse than the prior behavior in that case. + const vectorLedgerForParity = await this.storage.getCanonicalCounts?.() + const vectorParityTarget = + vectorLedgerForParity && !vectorLedgerForParity.suspect + ? vectorLedgerForParity.vectors.all + : metadataStats.totalEntries + if (hnswSize === vectorParityTarget) { checks.push({ name: 'index-parity', status: 'pass', - message: `HNSW (${hnswSize}) and metadata index (${metadataStats.totalEntries}) agree.`, - details: { hnswSize, metadataEntries: metadataStats.totalEntries, graphRelationships: graphSize } + message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) agree.`, + details: { + hnswSize, + vectoredNouns: vectorParityTarget, + metadataEntries: metadataStats.totalEntries, + graphRelationships: graphSize + } }) } else { - const drift = Math.abs(hnswSize - metadataStats.totalEntries) + const drift = Math.abs(hnswSize - vectorParityTarget) checks.push({ name: 'index-parity', - status: drift > Math.max(10, metadataStats.totalEntries * 0.01) ? 'fail' : 'warn', - message: `HNSW (${hnswSize}) and metadata (${metadataStats.totalEntries}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, - details: { hnswSize, metadataEntries: metadataStats.totalEntries, drift } + status: drift > Math.max(10, vectorParityTarget * 0.01) ? 'fail' : 'warn', + message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, + details: { + hnswSize, + vectoredNouns: vectorParityTarget, + metadataEntries: metadataStats.totalEntries, + drift + } }) } @@ -16066,6 +16275,79 @@ export class Brainy implements BrainyInterface { return !this.pluginRegistry.hasProvider('embeddings') } + /** + * @description LEG C of the zero-norm/unvector-door law — migrate a + * legacy zero-norm VFS root BEFORE the vector-leg open gate + * ({@link rebuildIndexesIfNeeded}'s `vectorCoverageGap` check) ever + * compares the canonical vectored-noun count against the vector index's + * size. A pre-fix store may have persisted the VFS root (the fixed + * all-zeros UUID) with a REAL all-zero placeholder vector — lawful inside + * brainy (`cosineDistance` treats a zero-norm operand as MAXIMUM distance, + * see {@link isZeroNormVector}'s JSDoc) but never indexed (the index belt + * refuses to insert a zero-norm vector) and never meant to cross an + * engine boundary. Left unmigrated, the canonical ledger still counts it + * as vectored while the vector index correctly holds nothing for it — a + * near-empty store whose ONLY vectored row is this zero-norm root reads + * "canonical vectored 1, index size 0" and throws + * `VectorIndexNotReadyError` at open, going DARK instead of serving. + * + * THE LIFECYCLE LAW: nothing at open may scale with brain size. This step + * is safe under that law BECAUSE the VFS root lives at a FIXED, + * well-known id (`00000000-0000-0000-0000-000000000000` — mirrors + * `VirtualFileSystem.VFS_ROOT_ID`; kept as a literal here, the same + * convention as the other reserved-root literals in this file and in + * `db/factLog.ts`/`db/portableGraph.ts` — `brainy.ts` cannot import + * `VirtualFileSystem.ts`, which itself imports `Brainy`) — this is ONE + * direct canonical read by id (`storage.getNoun`, the same O(1) + * fixed-path lookup {@link unvectorNounForRootMigration} itself uses + * internally), NEVER a listing or a walk over `entities/nouns/**`. An + * absent root (a store that has never used the VFS) is a no-op, no error. + * + * Runs UNCONDITIONALLY at every open, independent of whether a + * `VirtualFileSystem` is ever constructed this session — the vector-leg + * gate this fixes runs during Brainy's OWN init, before any + * `VirtualFileSystem` instance exists to run its own lazy migration at + * `doInitializeRoot()` (kept in place as the second line of defense for a + * VFS actually opened this session — belt AND suspenders, never either + * alone). + */ + private async migrateLegacyZeroNormVfsRootIfNeeded(): Promise { + const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + // TORN-TOLERANT: a torn root record is a recovery-walk healer's job + // (see tests/integration/recovery-walk-tolerance.test.ts — an init-time + // walk that meets a torn record narrates+counts, via the adapter's own + // loud floor at the read site, and heals PAST it; the open itself must + // still succeed), not this O(1) migration check's. Skip this open's + // migration attempt rather than aborting init(): this leg is a + // defensive EXTRA (the index belt + VirtualFileSystem's own + // doInitializeRoot() migration still stand as the other lines of + // defense), and it retries harmlessly at a later open once the root + // heals. + let root: HNSWNounWithMetadata | null + try { + root = await this.storage.getNoun(VFS_ROOT_ID) + } catch (err) { + if ((err as { code?: string }).code !== 'TORN_RECORD') throw err + prodLog.warn( + `[Brainy] open(): the VFS root's record is TORN — skipping the zero-norm root ` + + `migration check this open (the recovery walk is the healer; this migration ` + + `retries harmlessly once the root heals).` + ) + return + } + if (!root || !Array.isArray(root.vector) || root.vector.length === 0) return + if (!isZeroNormVector(root.vector)) return + const migrated = await this.unvectorNounForRootMigration(VFS_ROOT_ID) + if (migrated) { + prodLog.warn( + `[Brainy] open(): migrated the VFS root's legacy all-zero placeholder vector to ` + + `the unvectored shape (zero-norm vectors never cross an engine boundary) — run ` + + `before the vector-leg open gate compares canonical-vectored-count against the ` + + `vector index, so a near-empty store never reads a false coverage gap.` + ) + } + } + /** * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's * persisted vector from a real (non-empty) vector to the "unvectored" @@ -16075,13 +16357,18 @@ export class Brainy implements BrainyInterface { * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the * coverage ledger never silently drifts. * - * Exists SOLELY for the VFS root zero-norm migration (see - * `VirtualFileSystem.doInitializeRoot()`, which detects a persisted root - * whose vector is the legacy all-zero placeholder and calls this once per - * store). This is NOT a general-purpose "clear my vector" API — ordinary - * application data has no sanctioned path from vectored back to - * unvectored (`update()` refuses an empty vector as a dimension mismatch, - * by design). Never call this outside the VFS root migration. + * Exists SOLELY for the VFS root zero-norm migration, called from two + * sites that detect the same legacy shape (a persisted root whose vector + * is the legacy all-zero placeholder): {@link migrateLegacyZeroNormVfsRootIfNeeded} + * (this brain's own init sequence, BEFORE the vector-leg open gate — Leg + * C of the zero-norm/unvector-door law) and + * `VirtualFileSystem.doInitializeRoot()` (the second line of defense, for + * a VFS actually constructed this session). This is NOT the general- + * purpose unvector API — ordinary application data uses the sanctioned + * unvector DOOR instead (`update({ id, vector: [] })` / the same op inside + * `transact()`), which decrements the ledger and clears any pending + * deferred-embed marker inline; it does not call this method. Never call + * this outside a VFS root migration. * * Idempotent: a noun already unvectored (`vector.length === 0`) or absent * is a no-op — safe to call on every `init()`. diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 77e4f84d..1ed8f9ea 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -10,7 +10,7 @@ import { Vector, VectorDocument } from '../coreTypes.js' -import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' +import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' @@ -1768,6 +1768,33 @@ export class JsHnswVectorIndex implements VectorIndexProvider { // Process all nouns at once for (const nounData of result.items) { try { + // THE ZERO-NORM LAW / THE INDEX BELT — bulk-rebuild leg: a row + // holding no REAL vector (the "unvectored" `[]` shape — a + // deferred embed's stub, or a row the sanctioned unvector door + // rewrote) must never enter the index, mirroring the guard + // AddToVectorIndexOperation/ReplaceInVectorIndexOperation already + // enforce on the live transactional write paths. This matters + // HERE specifically: a row can be unvectored (canonical vector + // rewritten to `[]`) while its PERSISTED HNSW graph metadata + // (`getVectorIndexData` — level/connections) is still present + // from before the unvector, so `hnswData`'s mere presence below + // is not proof the row belongs in the index — only the + // canonical vector itself is authoritative. Checked against + // `nounData.vector` (the FULL loaded vector), never the + // to-be-truncated `noun.vector` below, so this holds regardless + // of the eager/lazy preload strategy chosen further down. + if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) { + continue + } + if (isZeroNormVector(nounData.vector)) { + prodLog.warn( + `[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` + + `zero-norm (a zero-norm vector is not a vector and never crosses an ` + + `engine boundary)` + ) + continue + } + // Load HNSW graph data for this entity const hnswData = await this.storage.getVectorIndexData(nounData.id) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 817e5e36..4f2a43b0 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -19,6 +19,7 @@ import { import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' import { prodLog } from '../../utils/logger.js' +import { isZeroNormVector } from '../../utils/distance.js' import { TornRecordError, isUnparseablePayloadError, @@ -2841,14 +2842,20 @@ export class FileSystemStorage extends BaseStorage { } /** - * Count canonical nouns holding a REAL (non-empty) vector — the vectored- - * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence - * cannot be decided from the id-directory listing alone: a deferred-embed - * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`) - * until its embed LANDS, so this walk reads every noun's `vectors.json` - * CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time - * legacy-counts.json derivation or a lost/corrupted counts.json recovery; - * the result is persisted so this scan never repeats. + * Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector — + * the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, + * presence cannot be decided from the id-directory listing alone: a + * deferred-embed noun's `vectors.json` EXISTS (written at `add()` time + * with `vector: []`) until its embed LANDS, so this walk reads every + * noun's `vectors.json` CONTENT — O(nouns) reads, not O(ids) listing. + * ZERO-NORM LAW: a real all-zero vector is not a vector — it never counts + * here either (Brainy's write paths normalize an explicit zero-norm + * vector to `[]` at write time, but a store created before that fix may + * still carry legacy all-zero rows on disk; this derivation must agree + * with the live ledger's definition of "vectored" regardless of when the + * row was written). Used ONLY for a one-time legacy-counts.json derivation + * or a lost/corrupted counts.json recovery; the result is persisted so + * this scan never repeats. */ private async scanVectoredNounCount(): Promise { const base = path.join(this.rootDir, 'entities', 'nouns') @@ -2862,7 +2869,12 @@ export class FileSystemStorage extends BaseStorage { for (const entry of ids) { if (!entry.isDirectory()) continue const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) - if (record && Array.isArray(record.vector) && record.vector.length > 0) { + if ( + record && + Array.isArray(record.vector) && + record.vector.length > 0 && + !isZeroNormVector(record.vector) + ) { vectored++ } } diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 606516ac..00790a4a 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -613,6 +613,17 @@ export function validateUpdateParams(params: UpdateParams): void { // null/undefined means "no new data was given". const hasData = params.data !== undefined && params.data !== null if ((params as UpdateParams & { deferEmbedding?: boolean }).deferEmbedding === true) { + if (params.vector && params.vector.length === 0) { + // The nonsensical combination Leg D of the zero-norm/unvector-door law + // refuses: `vector: []` is the SANCTIONED UNVECTOR DOOR — an explicit + // instruction to remove the vector NOW, never "please embed" — so it + // cannot be paired with a request to defer an embed. + throw new Error( + `update(): 'vector: []' (the unvector door) cannot be combined with ` + + `'deferEmbedding: true' — an unvector is an explicit instruction to remove ` + + `the vector now, not a request to defer an embed. Drop one of the two.` + ) + } if (params.vector) { throw new Error( `update(): deferEmbedding cannot be combined with an explicit 'vector' — ` + @@ -649,8 +660,16 @@ export function validateUpdateParams(params: UpdateParams): void { throw new Error(`invalid NounType: ${params.type}`) } - // Validate vector dimensions if provided - if (params.vector) { + // Validate vector dimensions if provided. A length-0 vector is the + // SANCTIONED UNVECTOR DOOR (see brainy.ts update()'s matching comment): an + // explicit `vector: []` — or a real all-zero vector, normalized to `[]` + // upstream by the zero-norm law — carries no dimension information, + // exactly like validateAddParams's identical exemption, so it is exempt + // from the dimension check rather than refused as a "0-dimensional + // vector". (The `deferEmbedding` combination above already refuses + // `vector: []` paired with `deferEmbedding: true` — an empty array is + // truthy, so that guard fires unconditionally on any explicit `vector`.) + if (params.vector && params.vector.length > 0) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) diff --git a/tests/integration/vfs-root-zero-norm.test.ts b/tests/integration/vfs-root-zero-norm.test.ts index 00260eb4..577ae7ee 100644 --- a/tests/integration/vfs-root-zero-norm.test.ts +++ b/tests/integration/vfs-root-zero-norm.test.ts @@ -18,9 +18,16 @@ * harness reproduces exactly what a pre-fix store looked like on disk) * is rewritten to `[]` on the next `init()`, the ledger is decremented * through the sanctioned path, and a second `init()` is a no-op. - * (c) THE BELT at the live provider-write seam: an entity added with an - * EXPLICIT all-zero vector (any dimension) still lands its canonical - * write, but the vector-index insert is refused loudly. + * (c) THE CANONICAL-WRITE NORMALIZATION (Leg A of the follow-up + * zero-norm/unvector-door fix): an entity added with an EXPLICIT + * all-zero vector (any dimension) is normalized to the "unvectored" + * `[]` shape BEFORE the canonical write, the ledger flag, and the index + * ops ever see it — the canonical write still succeeds, loudly, and the + * vector-index insert never happens (nothing to index). Supersedes the + * original "canonical keeps the zero vector, only the index refuses" + * shape: a downstream engine's health-report gate reads the canonical + * ledger directly, so leaving a zero-norm vector on the canonical side + * re-opened the exact false-attractor risk this whole fix closes. * (d) the migrated root never surfaces in `find()` results (it was already * hidden behind `visibility: 'system'` — this pin holds regardless). */ @@ -142,7 +149,7 @@ describe('VFS root zero-norm cure', () => { await brain.close() }) - it('(c) the live-write belt: an entity added with an explicit all-zero vector lands its canonical write, but the vector-index insert is refused loudly', async () => { + it('(c) canonical-write normalization: an entity added with an explicit all-zero vector persists UNVECTORED ([]), loudly, and never reaches the vector index', async () => { const dir = mkTmp() const brain = openBrain(dir) await brain.init() @@ -150,20 +157,26 @@ describe('VFS root zero-norm cure', () => { const warnSpy = vi.spyOn(prodLog, 'warn') const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const ledgerBefore = await brain.storage.getCanonicalCounts() const zeroVector = new Array(384).fill(0) const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) - // The canonical write succeeded — the entity is fully readable with its - // (real, all-zero) vector intact. + // The canonical write succeeded — but the zero-norm vector was + // normalized to the "unvectored" `[]` shape BEFORE it was persisted + // (Leg A: a zero-norm vector is not a vector — it never crosses an + // engine boundary, canonical side included). const entity = await brain.get(id, { includeVectors: true }) expect(entity).not.toBeNull() - expect(entity.vector).toEqual(zeroVector) + expect(entity.vector).toEqual([]) - // The vector-index insert was skipped — the index size never moved. + // Nothing to index — the vector-index size never moved, and the + // vectored-noun ledger never counted this row. const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size expect(sizeAfter).toBe(sizeBefore) + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - // The refusal was LOUD and named the entity. + // The normalization was LOUD and named the entity. const loudCall = warnSpy.mock.calls.find( (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') ) diff --git a/tests/integration/zero-norm-unvector-door.test.ts b/tests/integration/zero-norm-unvector-door.test.ts new file mode 100644 index 00000000..d3290010 --- /dev/null +++ b/tests/integration/zero-norm-unvector-door.test.ts @@ -0,0 +1,399 @@ +/** + * @module tests/integration/zero-norm-unvector-door + * @description THE SEAM LAW, GENERALIZED: "a zero-norm vector is not a + * vector — it never crosses an engine boundary." `tests/integration/ + * vfs-root-zero-norm.test.ts` pins the VFS-root-specific cure; this file + * pins the follow-up that generalizes it to every write path plus the + * sanctioned door for shedding a vector on purpose. + * + * Four legs pinned here: + * (A) THE CANONICAL WRITE NORMALIZES ZERO-NORM TO `[]` — `add()` (single and + * `transact()`) persists an explicit real all-zero vector as the + * "unvectored" `[]` shape, loudly, before the ledger flag/dimension + * pin/index ops ever see it. The canonical write still succeeds. + * (B) THE LEGACY DERIVATION IS ZERO-NORM-AWARE — a lost/corrupted + * `counts.json`'s one-time re-derivation walk excludes a persisted + * zero-norm row from the vectored-noun scalar, matching the live + * ledger's definition of "vectored". + * (C) THE LEGACY VFS ROOT MIGRATES AT OPEN, BEFORE THE GATE, IN O(1) — a + * store whose ONLY vectored row is a legacy all-zero VFS root opens + * clean (no `VectorIndexNotReadyError`), via one fixed-path read, never + * a listing. + * (D) THE UNVECTOR DOOR — `update({ id, vector: [] })` (and the same op + * inside `transact()`) is the sanctioned, idempotent way to shed a + * vector on purpose: ledger decrement exactly once, index removal, no + * re-embed, and a pending deferred-embed marker is cleared rather than + * left to re-vectorize the row later. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { prodLog } from '../../src/utils/logger.js' +import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' +import { BaseStorage } from '../../src/storage/baseStorage.js' + +const ROOT_ID = '00000000-0000-0000-0000-000000000000' + +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-zero-norm-unvector-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + vi.restoreAllMocks() + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) +} + +const countsPath = (root: string) => path.join(root, '_system', 'counts.json') + +describe('zero-norm canonical write + the sanctioned unvector door', () => { + it('(A1) add() with an explicit all-zero vector persists [], warns loudly, never indexes, and the ledger is unchanged', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const warnSpy = vi.spyOn(prodLog, 'warn') + + const zeroVector = new Array(384).fill(0) + const id = await brain.add({ data: 'zero-norm add', type: NounType.Document, vector: zeroVector }) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore) + + const loud = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') + ) + expect(loud).toBeDefined() + + await brain.close() + }) + + it('(A2) transact() add with an explicit all-zero vector — the same canonical normalization', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const warnSpy = vi.spyOn(prodLog, 'warn') + const zeroVector = new Array(384).fill(0) + const id = 'aaaaaaaa-0000-4000-8000-000000000001' + + await brain.transact([ + { op: 'add', id, type: NounType.Document, data: 'zero-norm transact add', vector: zeroVector } + ]) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + const loud = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') + ) + expect(loud).toBeDefined() + + await brain.close() + }) + + it('(B) the legacy counts.json derivation excludes a persisted zero-norm row from the vectored-noun scalar', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() + + // The VFS root alone (unvectored — []) — the floor. + const baseline = (await brain.storage.getCanonicalCounts()).vectors.all + + const realId = await brain.add({ data: 'a real vectored document', type: NounType.Document }) + + // Plant the legacy all-zero shape BY HAND: a genuine identity record + // (via add(), so it has real metadata) whose vector leg is then + // overwritten directly through the raw storage primitive — bypassing + // Leg A's canonical-write normalization entirely (brain.storage.saveNoun + // is not Brainy.add()/update()'s normalized path) — reproducing exactly + // what a pre-fix store could have persisted on disk. + const zeroId = await brain.add({ data: 'a legacy zero-norm document', type: NounType.Document }) + const zeroVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: zeroId, vector: zeroVector, connections: new Map(), level: 0 }) + + await brain.flush() + await brain.close() + + // Remove counts.json so the next open re-derives from scratch (the + // one-time legacy/lost-file derivation path — Leg B). + fs.rmSync(countsPath(dir), { force: true }) + + brain = openBrain(dir) + await brain.init() + const ledger = await brain.storage.getCanonicalCounts() + // Only realId counts; zeroId's persisted all-zero vector does not. + expect(ledger.vectors.all).toBe(baseline + 1) + + await brain.close() + }) + + it('(C) a legacy all-zero VFS root as the ONLY vectored row: open succeeds with no not-ready error, via an O(1) fixed-path read (no entities-tree readdir), and the ledger is 0 after open', async () => { + const dir = mkTmp() + + // SESSION 1 — build the legacy shape: the root is a REAL all-zero + // 384-dim vector, genuinely indexed and genuinely ledgered — exactly + // what a pre-fix store's root looked like on disk (see + // vfs-root-zero-norm.test.ts pin (b) for the identical harness). + // `index.addItem` is called directly (bypassing the transactional + // zero-norm belt) because the pre-fix code path had no such belt — this + // harness must match history, not the cure. No other entity is added, + // so the root is the store's ONLY vectored row. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.flush() + expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(1) + await brain.close() + + // SESSION 2 — reopen with a FAKE native vector provider that claims + // `serving: true` at `size()===0` (the exact shape a downstream + // engine's own health report can legitimately carry — same technique as + // tests/integration/vector-leg-open-build.test.ts). This is the ONLY + // codepath where the vector-leg open gate's FAIL-TYPED throw + // (VectorIndexNotReadyError) can fire; the built-in JS engine alone + // never reaches it (the size-heuristic branch just rebuilds instead) — + // so this is the faithful reproduction of the incident Leg C closes. + const readdirCalls: string[] = [] + const originalReaddir = fs.promises.readdir.bind(fs.promises) + vi.spyOn(fs.promises, 'readdir').mockImplementation(((...args: any[]) => { + readdirCalls.push(String(args[0])) + return (originalReaddir as any)(...args) + }) as any) + + // Spy at the PROTOTYPE level (BaseStorage.getNoun) — the new brain's + // storage instance does not exist until init() runs, so an + // instance-level spy cannot be installed beforehand. Records the + // readdir-call delta across the FIRST call made with the root id — + // Leg C's own fixed-path read — proving it needs no directory listing. + let readdirDeltaDuringRootRead: number | null = null + const originalGetNoun = BaseStorage.prototype.getNoun + vi.spyOn(BaseStorage.prototype, 'getNoun').mockImplementation(async function ( + this: unknown, + id: string + ) { + const before = readdirCalls.length + const result = await originalGetNoun.call(this as BaseStorage, id) + if (id === ROOT_ID && readdirDeltaDuringRootRead === null) { + readdirDeltaDuringRootRead = readdirCalls.length - before + } + return result + }) + + brain = openBrain(dir) + brain.use({ + name: 'fake-native-vector-unledgered-coverage', + activate: async (ctx: any) => { + ctx.registerProvider('vector', (config: any, distance: any, options: any) => { + const real = new JsHnswVectorIndex(config, distance, options) + let rebuilt = false + const originalRebuild = real.rebuild.bind(real) + ;(real as any).rebuild = async (...args: any[]) => { + const r = await originalRebuild(...args) + rebuilt = true + return r + } + const originalSize = real.size.bind(real) + ;(real as any).size = () => (rebuilt ? originalSize() : 0) + ;(real as any).healthReport = () => ({ + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 0, + generation: 1, + unledgered: ['vector-coverage'] + }) + return real + }) + return true + } + }) + + // Must NOT throw VectorIndexNotReadyError (or anything else) — a + // near-empty store whose only vectored row is the zero-norm root must + // never go dark. + await brain.init() + + const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) + expect(migratedRoot.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(0) + + expect(readdirDeltaDuringRootRead).toBe(0) + + await brain.close() + }) + + describe('the sanctioned unvector door', () => { + it('(D1) update({ id, vector: [] }) unvectors a real vectored row — canonical [], removed from the index, ledger decremented by exactly 1, no embed call', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + await brain.flush() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + + const embedSpy = vi.spyOn(brain, 'embed') + await brain.update({ id, vector: [] }) + expect(embedSpy).not.toHaveBeenCalled() + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore - 1) + + await brain.close() + }) + + it('(D2) idempotent: a second update({ id, vector: [] }) on an already-unvectored row is a true no-op — no error, no further decrement', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + await brain.flush() + + await brain.update({ id, vector: [] }) + const ledgerAfterFirst = await brain.storage.getCanonicalCounts() + + await brain.update({ id, vector: [] }) + const ledgerAfterSecond = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfterFirst.vectors.all) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + await brain.close() + }) + + it('(D3) a PENDING deferred-embed row: the unvector door clears the marker; awaitPendingEmbeds() then leaves it unvectored', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + // Prevent the background worker from ever actually running — it is + // fire-and-forget from add(), and a real run would race this test's + // own assertions (see tests/integration/vector-leg-open-build.test.ts + // for the same concern). This isolates exactly the marker-clearing + // behavior under test. + vi.spyOn(brain as any, 'kickEmbedWorker').mockImplementation(() => {}) + + const id = await brain.add({ + data: 'deferred content, never embedded', + type: NounType.Document, + deferEmbedding: true + }) + expect(brain.pendingEmbedCount()).toBe(1) + + const warnSpy = vi.spyOn(prodLog, 'warn') + await brain.update({ id, vector: [] }) + + expect(brain.pendingEmbedCount()).toBe(0) + const clearedWarn = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('pending') + ) + expect(clearedWarn).toBeDefined() + + // The barrier must not hang and must not re-vectorize the row — the + // worker (still mocked to a no-op) never runs again. + await brain.awaitPendingEmbeds() + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + await brain.close() + }) + + it('(D4) update({ vector: [], deferEmbedding: true }) is a typed refusal — the unvector door cannot be paired with a deferred embed', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + const before = await brain.get(id, { includeVectors: true }) + + await expect( + brain.update({ id, vector: [], deferEmbedding: true }) + ).rejects.toThrow(/unvector door/i) + + // Refused before any write — the row is untouched. + const after = await brain.get(id, { includeVectors: true }) + expect(after.vector).toEqual(before.vector) + + await brain.close() + }) + + it('(D5) the transact() twin of the unvector door decrements the ledger exactly once, and is idempotent on a second call', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document for transact unvector', type: NounType.Document }) + await brain.flush() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + + await brain.transact([{ op: 'update', id, vector: [] }]) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore - 1) + + // Idempotent through transact() too. + await brain.transact([{ op: 'update', id, vector: [] }]) + const ledgerAfterSecond = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfter.vectors.all) + + await brain.close() + }) + }) +}) From 6f93108648bf15e90958bbe02f7c7857c900d387 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 14:47:48 -0700 Subject: [PATCH 03/42] chore(release): 10.4.2-rc.1 --- CHANGELOG.md | 17 +++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec925642..0b154af5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) + +- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) +- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) +- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1) +- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4) +- Merge branch 'next/enumeration-identity-rekey' (204d74c1) +- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16) +- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a) +- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab) +- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9) +- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245) +- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee) +- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb) +- chore(release): 10.4.1-rc.1 (7870dc40) + + ### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) diff --git a/package-lock.json b/package-lock.json index 70635835..889c67c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 4125b073..214feeac 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From a082e0efdd502941eb1baa2fa3d1ac30b0d0b095 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 15:07:25 -0700 Subject: [PATCH 04/42] docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy --- RELEASES.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index bc400412..6de11244 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,95 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.4.2 — 2026-08-28 (a zero-norm vector is not a vector) + +**This is the last release of the MIT engine under the `@soulcraft/brainy` name.** +The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API, +client library, types and protocol, an openly specified canonical format, and the TypeScript +reference engine, scoped honestly as a single-node engine for stores up to roughly one +million rows. The `@soulcraft/brainy` name passes to the native engine, **Brainy**, at a +major version bump; that engine implements the same API over the same open format at +production scale, requires a license, and refuses loudly without one. Nothing changes +for existing installs until that major ships; the move is announced with it. + +Six fixes, one law: a vector with no magnitude carries no information, so it must +never reach a vector index — in any engine — and the canonical store must say so. + +- **The permanently-unvectored row.** `add({ ..., vector: [] })` (and the same item + shape in `addMany` / `transact`) is now the sanctioned "no vector" row: persisted + with an empty vector leg, never embedded, never indexed, counted as unvectored in + the canonical ledger. Metadata-only rows — telemetry tallies, counters, plumbing — + no longer need a placeholder vector and never enter the vector leg. `vector: []` + together with `deferEmbedding: true` is refused with a typed error (a supplied + vector has nothing to defer). Previously `vector: []` threw a dimension error. +- **The unvector door.** `update({ id, vector: [] })` (and its `transact()` twin) is + the sanctioned way to strip a vector from an existing row: canonical vector → `[]`, + removal from the vector index, the vectored ledger decremented exactly once — and + idempotent, so a resumed cleanup pass may simply re-issue. It never re-embeds, and + it clears a pending deferred-embed marker durably so the background worker cannot + re-vector the row later. Note that a rebuild never sheds vectors (it re-derives the + index from canonical rows); shedding historical vectors needs this door. +- **Zero-norm vectors are normalized at the write.** An explicit all-zero vector on + any write path is persisted as unvectored (`[]`) with one warning naming the row; + the vector-index operations keep their own refusal as a second line. The engine's + own VFS root, which used to persist a deliberate all-zero placeholder (harmless + under cosine distance, a false attractor under a downstream engine's + squared-euclidean serving — a production incident this week), is now created + unvectored, and an existing store's legacy root is migrated on open by a single + fixed-path read before the health gate runs — never a walk. +- **Enumeration keys on the identity record.** `getNouns()` / `getVerbs()` and the + cursor walks behind them enumerate by the metadata record, the same key the + canonical ledger counts by — previously the walk keyed on the vector file, so a + row holding metadata but no vector was counted yet never yielded (a permanent + "missing" phantom in coverage math), while an orphaned vector-only directory + could be yielded as a phantom id. The recovery fold also never deletes an existing + vector when it replays a metadata-only after-image (preserve-if-absent). One + documented gap remains: a verb's endpoints live only in its vector leg, so a + metadata-only verb is counted and loudly skipped, never fabricated — the fix is a + canonical-format change and lands with the open format. +- **The ledger's one-time derivation counts identity records.** Stores upgraded from + pre-ledger versions derived their ALL-visibility scalars once by counting id + directories, which included ghost and scar containers left by an old partial-delete + defect — an inflated denominator whose coverage row could never reach exact. The + derivation now counts only directories holding a metadata record, `counts.json` + carries a derivation-rule stamp, and a ledger derived under the old rule is marked + `suspect` at open (one O(1) field read, one warning) so the online `repairIndex()` + path clears it with a real recount. +- **The vector index refuses what it cannot hold.** `rebuild()` skips unvectored and + zero-norm rows (one summary line), re-pins the vector dimension from the first real + vector after a restart (previously a restart left the pin unset, so a wrong-length + insert became the new pin instead of being rejected), and `addItem` / `updateItem` + throw a typed `EmptyVectorIndexError` on a length-0 vector instead of ever storing + a vector-less node. +- **Smaller:** a failing plugin activation now rethrows with the original error as + `cause` (the originating file and line survive to the caller's log); build + generators stamp from the repository history of their inputs instead of wall clock, + so two builds of the same tree are byte-identical. + +Adoption: one restart, paired with its native-engine release. The first open of an +existing store runs the legacy-root migration (one narrated line) and, on stores that +upgraded from pre-ledger versions, marks the ledger suspect until the next sanctioned +recount — no rebuild in either case. + +## v10.4.1 — 2026-08-26 (reads refuse per family; an unchanged write never re-embeds) + +Two production defects from the same week, fixed together as a patch to 10.4.0. + +- **The read gate is per family.** A read now refuses only when the index family it + actually consults is unhealthy: a metadata filter is served while the vector leg is + rebuilding; a semantic query is refused only by the vector family; a graph + traversal only by the graph family. Previously any unhealthy family refused every + read on the brain — under a long vector rebuild, a production deployment's + metadata-only reads were refused for the duration, and the retries became a write + pump of their own. +- **Unchanged data never re-embeds.** `update()` compares the incoming `data` + structurally with the stored record; an update carrying identical data (a common + shape for periodic upserts) no longer embeds again and no longer churns the vector + leg. Previously every such update re-embedded and re-inserted, which under load + saturated the vector index with near-identical vectors. + +Adoption: one restart, paired with its native-engine release. + ## v10.4.0 — 2026-08-25 (the health report has a name) Three related cures, one root cause: an index deciding whether it could be trusted From 1f34fc6ce496ef9a359be04153da1397191f50b3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 15:14:03 -0700 Subject: [PATCH 05/42] chore(release): 10.4.2 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b154af5..d13a2d66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) + +- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) + + ### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) - Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) diff --git a/package-lock.json b/package-lock.json index 889c67c8..fb88681f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.2", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 214feeac..17983c19 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 9f248b24952ac061fc05844a57541c8562ba35f1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 16:56:45 -0700 Subject: [PATCH 06/42] =?UTF-8?q?docs(releases):=2010.4.3=20=E2=80=94=20Op?= =?UTF-8?q?en=20Brainy's=20first=20release=20under=20the=20new=20name,=20s?= =?UTF-8?q?ame=20engine=20as=2010.4.2;=20The=20Source=20is=20the=20one=20r?= =?UTF-8?q?egistry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index 6de11244..b89beba0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,7 +31,30 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v10.4.2 — 2026-08-28 (a zero-norm vector is not a vector) +## v10.4.3 — 2026-08-27 (Open Brainy's first release) + +**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for +byte — only the name, the registry, and the pointers changed.** Install: + +```bash +npm install @soulcraftlabs/brainy +``` + +with the registry line in your `.npmrc` (anonymous read): + +``` +@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/ +``` + +- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the + npmjs republish step is retired from the release rail. Existing npmjs versions of + `@soulcraft/brainy` stay as they are and receive no new versions. +- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects. +- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one + install-line change (`@soulcraft/brainy` → `@soulcraftlabs/brainy`), which downstream + applications make together with their native-engine bump. + +## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector) **This is the last release of the MIT engine under the `@soulcraft/brainy` name.** The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API, From a99b1e83c4548d6dec78ee1840267678a1d68dc1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 17:07:09 -0700 Subject: [PATCH 07/42] chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepares the repo for its new home at soulcraftlabs/open-brainy ahead of the Forgejo transfer: package name, publish registry, release script, and every install/import reference across docs, src, tests, examples, and integrations now point at @soulcraftlabs/brainy on The Source. The npmjs storefront leg and byte-identity pair verification are stripped from the release script — The Source is now the only publish target. README gains an Open Brainy explainer and a registry note for consumers. @soulcraft/brainy 10.4.2 was the last release under the old name. --- .claude/skills/architecture.md | 2 +- .forgejo/workflows/publish-source.yml | 14 +-- CLAUDE.md | 4 +- README.md | 16 ++-- SECURITY.md | 2 +- bin/brainy-ts.js | 2 +- bun.lock | 2 +- docs/DEVELOPER_LEARNING_PATH.md | 12 +-- docs/FIND_SYSTEM.md | 2 +- docs/MIGRATION-V3-TO-V4.md | 12 +-- docs/PLUGINS.md | 24 ++--- docs/PRODUCTION_SERVICE_ARCHITECTURE.md | 4 +- docs/README.md | 2 +- docs/RELEASE-GUIDE.md | 2 +- docs/SCALING.md | 4 +- docs/api/README.md | 10 +- .../architecture/data-storage-architecture.md | 2 +- docs/architecture/finite-type-system.md | 6 +- .../multiprocess-storage-mixin.md | 2 +- docs/architecture/noun-verb-taxonomy.md | 4 +- docs/architecture/zero-config.md | 2 +- docs/concepts/field-addressing.md | 2 +- docs/concepts/index-health.md | 2 +- docs/concepts/storage-adapters.md | 16 ++-- docs/guides/aggregation.md | 2 +- docs/guides/framework-integration.md | 24 ++--- docs/guides/import-anything.md | 2 +- docs/guides/import-progress-examples.md | 2 +- docs/guides/import-quick-reference.md | 4 +- docs/guides/inspection.md | 2 +- docs/guides/installation.md | 14 +-- docs/guides/migration-3.36.0.md | 10 +- docs/guides/model-loading.md | 2 +- docs/guides/namespace-migration.md | 4 +- docs/guides/nextjs-integration.md | 18 ++-- docs/guides/optimistic-concurrency.md | 4 +- docs/guides/quick-start.md | 6 +- docs/guides/standard-import-progress.md | 6 +- docs/guides/storage-adapters.md | 4 +- docs/guides/subtypes-and-facets.md | 4 +- docs/guides/upgrading-7-to-8.md | 6 +- docs/guides/vue-integration.md | 6 +- docs/neural-extraction.md | 16 ++-- docs/transactions.md | 6 +- docs/universal-display-augmentation.md | 2 +- docs/vfs/PROJECTION_STRATEGY_API.md | 12 +-- docs/vfs/QUICK_START.md | 10 +- docs/vfs/README.md | 4 +- docs/vfs/ROADMAP.md | 6 +- docs/vfs/SEMANTIC_VFS.md | 2 +- docs/vfs/VFS_API_GUIDE.md | 4 +- docs/vfs/VFS_CORE.md | 4 +- docs/vfs/VFS_GRAPH_TYPES.md | 2 +- docs/vfs/VFS_INITIALIZATION.md | 6 +- docs/vfs/building-file-explorers.md | 6 +- examples/bluesky-distributed-setup.js | 2 +- examples/monitor-cache-performance.ts | 2 +- integrations/README.md | 8 +- integrations/google-sheets/README.md | 4 +- package-lock.json | 4 +- package.json | 11 ++- scripts/release.sh | 93 ++++--------------- src/brainy.ts | 6 +- src/db/errors.ts | 2 +- src/embeddings/wasm/modelLoader.ts | 8 +- src/errors/notFound.ts | 2 +- src/integrations/index.ts | 2 +- src/mcp/README.md | 4 +- src/plugin.ts | 4 +- src/types/brainy.types.ts | 2 +- src/types/reservedFields.ts | 2 +- src/utils/brainyTypes.ts | 2 +- src/utils/version.ts | 2 +- tests/unit/brainy/migration-deference.test.ts | 4 +- 74 files changed, 230 insertions(+), 284 deletions(-) diff --git a/.claude/skills/architecture.md b/.claude/skills/architecture.md index de046b17..4de3c287 100644 --- a/.claude/skills/architecture.md +++ b/.claude/skills/architecture.md @@ -2,7 +2,7 @@ ## What Is Brainy -@soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. +@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. ## Core Architecture diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml index a08875ae..58cb1d30 100644 --- a/.forgejo/workflows/publish-source.yml +++ b/.forgejo/workflows/publish-source.yml @@ -32,7 +32,7 @@ jobs: run: | set -eo pipefail - SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" + SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" VERSION="$(node -p "require('./package.json').version")" # The dist-tag follows the version: a prerelease (any hyphen — # 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' — @@ -43,13 +43,13 @@ jobs: case "$VERSION" in *-*) NPM_TAG="rc" ;; esac - echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." + echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." TMPRC="$(mktemp)" chmod 600 "$TMPRC" { - echo "@soulcraft:registry=${SOURCE_NPM_REG}" - echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}" + echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}" } > "$TMPRC" # The release script bumps package.json's version before it tags, so @@ -64,7 +64,7 @@ jobs: # exit code: a benign duplicate publish (a prior run, or a mirror, already # landed this exact version) reports failure even though the registry # already holds the right content. - LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" + LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" rm -f "$TMPRC" if [ "$LANDED_VERSION" != "$VERSION" ]; then @@ -73,7 +73,7 @@ jobs: fi if [ "$PUBLISH_OK" = true ]; then - echo "Published and verified @soulcraft/brainy@${VERSION} on The Source registry." + echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry." else - echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." + echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." fi diff --git a/CLAUDE.md b/CLAUDE.md index 6acd0e0d..56df0b72 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md` **Brainy's current open actions:** None. MIT open-source — no platform-specific actions. -**Current version:** run `npm view @soulcraft/brainy version` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` +**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` --- ## Project Overview -Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license. +Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license. ## Getting Started diff --git a/README.md b/README.md index ca558340..47a9123a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- Brainy + Brainy

Brainy

@@ -11,8 +11,8 @@

- npm version - npm downloads + Package on The Source + Repository CI Documentation MIT License @@ -30,6 +30,8 @@ --- +**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license. + Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together: | You write | Brainy indexes it as | You query it with | @@ -45,12 +47,14 @@ It runs **inside your process** — no server, no Docker, nothing to operate — ## Quick start ```bash -bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended -npm install @soulcraft/brainy # Node.js ≥ 22 +bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended +npm install @soulcraftlabs/brainy # Node.js ≥ 22 ``` +> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read). + ```javascript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() // in-memory; one line swaps to disk await brain.init() diff --git a/SECURITY.md b/SECURITY.md index 1f3c4732..91d40d49 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -30,7 +30,7 @@ commit to backporting fixes to unsupported lines. ## Scope -This policy covers the `@soulcraft/brainy` package itself — the code in +This policy covers the `@soulcraftlabs/brainy` package itself — the code in this repository. If you're evaluating a deployment that also uses `@soulcraft/cor`, report issues in that package the same way, to the same address; we'll route internally. diff --git a/bin/brainy-ts.js b/bin/brainy-ts.js index 4e9aedb8..90a35e98 100644 --- a/bin/brainy-ts.js +++ b/bin/brainy-ts.js @@ -3,7 +3,7 @@ /** * Modern TypeScript CLI Runner * - * This is the entry point after npm install @soulcraft/brainy + * This is the entry point after npm install @soulcraftlabs/brainy * It runs the compiled TypeScript CLI code */ diff --git a/bun.lock b/bun.lock index c31b3865..1e3e66e2 100644 --- a/bun.lock +++ b/bun.lock @@ -3,7 +3,7 @@ "configVersion": 0, "workspaces": { "": { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", "@azure/identity": "^4.0.0", diff --git a/docs/DEVELOPER_LEARNING_PATH.md b/docs/DEVELOPER_LEARNING_PATH.md index 4134ae63..b2d22fb1 100644 --- a/docs/DEVELOPER_LEARNING_PATH.md +++ b/docs/DEVELOPER_LEARNING_PATH.md @@ -25,13 +25,13 @@ ### Prerequisites ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Your First Neural Database ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // Step 1: Create and initialize Brainy const brain = new Brainy({ @@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou ### Building a Knowledge Graph ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**. ### Triple Intelligence in Action ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in ### Files as Intelligent Entities ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**. ### Production-Ready Deployment ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // 1. PRODUCTION STORAGE - Filesystem with off-site snapshots console.log('Initializing production storage...\n') diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 1cc38ce9..77fbbd79 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -1217,7 +1217,7 @@ where: { await brain.find({ type: 'Document' }) // ✅ Correct: Use NounType enum -import { NounType } from '@soulcraft/brainy' +import { NounType } from '@soulcraftlabs/brainy' await brain.find({ type: NounType.Document }) // ❌ Error: Operator not recognized diff --git a/docs/MIGRATION-V3-TO-V4.md b/docs/MIGRATION-V3-TO-V4.md index 29c409ac..680b6928 100644 --- a/docs/MIGRATION-V3-TO-V4.md +++ b/docs/MIGRATION-V3-TO-V4.md @@ -153,13 +153,13 @@ brainy-data/ ### Step 1: Update Brainy Package ```bash -npm install @soulcraft/brainy@latest +npm install @soulcraftlabs/brainy@latest ``` **Check your version:** ```bash -npm list @soulcraft/brainy -# Should show: @soulcraft/brainy@4.0.0 +npm list @soulcraftlabs/brainy +# Should show: @soulcraftlabs/brainy@4.0.0 ``` ### Step 2: No Code Changes Required! ✅ @@ -374,7 +374,7 @@ If you encounter issues, you can rollback: ```bash # Reinstall v3 -npm install @soulcraft/brainy@^3.50.0 +npm install @soulcraftlabs/brainy@^3.50.0 # Restart application ``` @@ -389,7 +389,7 @@ rm -rf ./data cp -r ./data-backup ./data # Reinstall v3 -npm install @soulcraft/brainy@^3.50.0 +npm install @soulcraftlabs/brainy@^3.50.0 ``` ## Common Migration Scenarios @@ -539,7 +539,7 @@ console.log('Storage type:', status.type) **Migration Checklist:** - ✅ Backup data -- ✅ Update npm package (`npm install @soulcraft/brainy@latest`) +- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`) - ✅ Restart application (automatic migration) - ✅ Verify data integrity - ✅ Enable lifecycle policies diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index 3078c239..238d2252 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement ### 1. Implement the `BrainyPlugin` interface ```typescript -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' const myPlugin: BrainyPlugin = { name: 'my-brainy-plugin', // Must be unique (typically your npm package name) @@ -90,7 +90,7 @@ await brain.init() **Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`: ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import myPlugin from './my-plugin.js' const brain = new Brainy() @@ -272,10 +272,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor #### `cache` **Type:** `UnifiedCache` -Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`). +Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`). ```typescript -import type { UnifiedCache } from '@soulcraft/brainy/internals' +import type { UnifiedCache } from '@soulcraftlabs/brainy/internals' context.registerProvider('cache', myNativeCache) ``` @@ -325,8 +325,8 @@ Plugins can register custom storage backends that users reference by name. ### Implementing a Storage Adapter ```typescript -import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin' -import type { StorageAdapter } from '@soulcraft/brainy' +import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin' +import type { StorageAdapter } from '@soulcraftlabs/brainy' class MyStorageAdapter implements StorageAdapter { async init(): Promise { /* ... */ } @@ -360,9 +360,9 @@ Brainy provides three entry points for plugin developers: | Import Path | Contents | Stability | |-------------|----------|-----------| -| `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) | -| `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | -| `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | +| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) | +| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | +| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | ## Diagnostics @@ -440,7 +440,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations ```typescript // simd-distance-plugin/src/plugin.ts -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' // Hypothetical native module import { simdCosineDistance } from './native.js' @@ -470,7 +470,7 @@ export default simdDistancePlugin "main": "./dist/plugin.js", "types": "./dist/plugin.d.ts", "peerDependencies": { - "@soulcraft/brainy": ">=7.0.0" + "@soulcraftlabs/brainy": ">=7.0.0" } } ``` @@ -478,7 +478,7 @@ export default simdDistancePlugin Usage: ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ plugins: ['brainy-simd-distance'] }) await brain.init() diff --git a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md index 4568cd31..ad4a4a40 100644 --- a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md +++ b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md @@ -54,7 +54,7 @@ After 40 API calls: ```typescript // server.ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // SINGLETON INSTANCE let brainInstance: Brainy | null = null @@ -174,7 +174,7 @@ process.on('SIGTERM', async () => { ```typescript // server.ts - Clean Bun implementation -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain: Brainy | null = null diff --git a/docs/README.md b/docs/README.md index 3290001f..ddb37d20 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,7 +5,7 @@ ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/RELEASE-GUIDE.md b/docs/RELEASE-GUIDE.md index 94c6a6cb..4b120bd8 100644 --- a/docs/RELEASE-GUIDE.md +++ b/docs/RELEASE-GUIDE.md @@ -99,7 +99,7 @@ Examples: ```bash # 1. Deprecate wrong version on npm -npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y" +npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y" # 2. Fix version in package.json # 3. Republish correct version diff --git a/docs/SCALING.md b/docs/SCALING.md index e9ae1136..054d2096 100644 --- a/docs/SCALING.md +++ b/docs/SCALING.md @@ -13,7 +13,7 @@ ### In-Memory ```typescript -import Brainy from '@soulcraft/brainy' +import Brainy from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) ``` @@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the -open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native +open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`. `find()` query latency, p50 / p95 (200 queries each): diff --git a/docs/api/README.md b/docs/api/README.md index 82f48c91..ba49ff48 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -24,7 +24,7 @@ next: ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() // Zero config! await brain.init() // VFS auto-initialized! @@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization ### Db API errors -All exported from `@soulcraft/brainy`: +All exported from `@soulcraftlabs/brainy`: | Error | Thrown by | Meaning | |---|---|---| @@ -1918,11 +1918,11 @@ isn't serving throws instead of rebuilding mid-query: | `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving | | `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving | -All three are exported from `@soulcraft/brainy`. Catch them to distinguish +All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish "index not ready" from a genuine empty result: ```typescript -import { MetadataIndexNotReadyError } from '@soulcraft/brainy' +import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy' try { const rows = await brain.find({ where: { status: 'active' } }) @@ -2208,7 +2208,7 @@ For the full taxonomy with all 169 types and their descriptions, see: - **📖 Documentation:** [Full Documentation](../) - **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues) - **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions) -- **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy) +- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy) - **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy) --- diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 48064757..83b9e23a 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -268,7 +268,7 @@ locks/_flush_responses/ # writer answers with .ack | **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) | A pluggable index provider (the 8.0 plugin contract in -`@soulcraft/brainy/plugin`) may replace any of the JS implementations; the +`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the persisted formats above are contract-bound so JS and native implementations can interleave on the same directory. diff --git a/docs/architecture/finite-type-system.md b/docs/architecture/finite-type-system.md index 76492ee5..48a8b1fe 100644 --- a/docs/architecture/finite-type-system.md +++ b/docs/architecture/finite-type-system.md @@ -126,7 +126,7 @@ class TypeAwareMetadataIndex { **The Design**: Specify types clearly in your API calls: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' // Add entity with explicit type await brain.add({ @@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation { **Brainy's Approach**: Extract **typed** concepts: ```typescript -import { NaturalLanguageProcessor } from '@soulcraft/brainy' +import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy' const nlp = new NaturalLanguageProcessor() const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco") @@ -382,7 +382,7 @@ import { getVerbTypes, BrainyTypes, suggestType -} from '@soulcraft/brainy' +} from '@soulcraftlabs/brainy' // Get all available noun types const nounTypes = getNounTypes() diff --git a/docs/architecture/multiprocess-storage-mixin.md b/docs/architecture/multiprocess-storage-mixin.md index 46f98398..1593bf8f 100644 --- a/docs/architecture/multiprocess-storage-mixin.md +++ b/docs/architecture/multiprocess-storage-mixin.md @@ -127,7 +127,7 @@ For reference, a clean migration path: `isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for build/install artifact protection. 5. Document the new contract in `concepts/storage-adapters.md`. -6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by +6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by plugins. Estimated work: ~half a day of code, ~2 hours of doc/example updates, diff --git a/docs/architecture/noun-verb-taxonomy.md b/docs/architecture/noun-verb-taxonomy.md index 286464be..3dac6892 100644 --- a/docs/architecture/noun-verb-taxonomy.md +++ b/docs/architecture/noun-verb-taxonomy.md @@ -20,7 +20,7 @@ next: Every example on this page is written against the real Brainy 8.0 API. The setup is always the same: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu - **Multi-hop Graph Traversals = Relationship Complexity** - **Result: Model data across virtually any industry** -Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. +Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. ## The Power of Standardization: Universal Interoperability diff --git a/docs/architecture/zero-config.md b/docs/architecture/zero-config.md index d42d6784..a35e6416 100644 --- a/docs/architecture/zero-config.md +++ b/docs/architecture/zero-config.md @@ -35,7 +35,7 @@ constructor and `init()`. ## Instant Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // That's it. No config needed. const brain = new Brainy() diff --git a/docs/concepts/field-addressing.md b/docs/concepts/field-addressing.md index c459021b..d24dd66b 100644 --- a/docs/concepts/field-addressing.md +++ b/docs/concepts/field-addressing.md @@ -167,7 +167,7 @@ await brain.find({ orderBy: 'createdAt' }) `UnresolvableFieldError` is exported from the package root: ```typescript -import { UnresolvableFieldError } from '@soulcraft/brainy' +import { UnresolvableFieldError } from '@soulcraftlabs/brainy' try { await brain.find({ orderBy: 'createdAt' }) diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md index 96199f11..923267df 100644 --- a/docs/concepts/index-health.md +++ b/docs/concepts/index-health.md @@ -95,7 +95,7 @@ catchable error naming the reason: | `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" | | `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" | -All three are exported from `@soulcraft/brainy`. Catch them where your application +All three are exported from `@soulcraftlabs/brainy`. Catch them where your application needs to distinguish "this index isn't ready yet" from "there's genuinely nothing here" — a health dashboard, a retry policy, an operator alert. The fix is always the same: reconcile the index, either by reopening the brain (which brings every diff --git a/docs/concepts/storage-adapters.md b/docs/concepts/storage-adapters.md index af6d068f..82aa01e8 100644 --- a/docs/concepts/storage-adapters.md +++ b/docs/concepts/storage-adapters.md @@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from to call `acquireWriterLock()` at init. ```typescript -import { FileSystemStorage } from '@soulcraft/brainy' +import { FileSystemStorage } from '@soulcraftlabs/brainy' export class MmapFileSystemStorage extends FileSystemStorage { public supportsMultiProcessLocking(): boolean { @@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom network backend), extend `BaseStorage` directly: ```typescript -import { BaseStorage } from '@soulcraft/brainy' +import { BaseStorage } from '@soulcraftlabs/brainy' export class MyCloudStorage extends BaseStorage { // BaseStorage's default no-op implementations of the multi-process @@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`, `hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM import (verify in your plugin's `dist/`: `import { FileSystemStorage } from -'@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype +'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype chain at runtime resolves to whatever Brainy version your consumer has installed. @@ -109,8 +109,8 @@ installed. the prototype chain at the consumer-app level: - **Stale `node_modules`** — a lingering install from before the consumer - upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but - `node_modules/@soulcraft/brainy` is still 7.20.x. + upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but + `node_modules/@soulcraftlabs/brainy` is still 7.20.x. - **Lockfile drift** — `bun.lockb` / `package-lock.json` pins a brainy version older than the package.json range, and `bun install` honors the lockfile. @@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint: methods on its prototype chain. Writer locking and the flush-request RPC are disabled for this directory. Likely fix: clean install (`rm -rf node_modules bun.lockb && bun install`) or rebuild your container image to refresh -`@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. +`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. ``` ## Authoring a new storage adapter — minimum checklist @@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh install time — fix install, not your plugin. 6. **Pin your peer dep generously.** `"peerDependencies": { - "@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin + "@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin to an exact patch unless you're tracking a known regression. ## Future direction @@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change. heartbeat semantics, what the lock protects. - [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the read-only mode. -- `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the +- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the authoritative type signatures for every method this page references. diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index 11d86ec8..616c8fc4 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -22,7 +22,7 @@ they share a single scan. ## Quick Start ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/framework-integration.md b/docs/guides/framework-integration.md index 984466c5..8f85da00 100644 --- a/docs/guides/framework-integration.md +++ b/docs/guides/framework-integration.md @@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server: -- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'` +- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'` - **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node - **Cleaner code**: No browser polyfills, no conditional client/server imports - **Better DX**: One instance shared across your server routes @@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers ### Install Brainy ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Integration ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Run on the server (API route, server component, backend service) // new Brainy() auto-detects filesystem persistence on Node @@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod ```javascript // lib/brain.server.js -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests: ```javascript // server/brain.js (server-only module) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun): ```typescript // server: api/search -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() // auto-detects filesystem persistence on Node await brain.init() @@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or ```javascript // lib/brain.server.js (imported only by server code) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul ```javascript // src/lib/server/brain.js (server-only — note the .server suffix) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -432,7 +432,7 @@ import { defineConfig } from 'vite' export default defineConfig({ ssr: { - external: ['@soulcraft/brainy'] + external: ['@soulcraftlabs/brainy'] } }) ``` @@ -440,7 +440,7 @@ export default defineConfig({ ```javascript // rollup.config.js (server bundle) export default { - external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto'] + external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto'] } ``` @@ -466,7 +466,7 @@ export async function load({ url }) { ```javascript // For build-time usage (runs in Node during the build) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' export async function generateStaticProps() { const brain = new Brainy({ @@ -513,7 +513,7 @@ export async function generateStaticProps() { ### Issue: Large client bundle size **Cause**: A client module is pulling in Brainy. -**Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle. +**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle. ### Issue: SSR hydration mismatch **Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup. diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index b1bb15ef..ffabe55c 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING: ## The Ultimate Simplicity ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/import-progress-examples.md b/docs/guides/import-progress-examples.md index 66f50713..18c3cb9a 100644 --- a/docs/guides/import-progress-examples.md +++ b/docs/guides/import-progress-examples.md @@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats** ### Basic Progress Tracking ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import * as fs from 'fs' const brain = await Brainy.create() diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index 7837d49e..3bc26dae 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -7,7 +7,7 @@ ## Basic Import ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -187,7 +187,7 @@ await brain.import(file, { ## Complete Example ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import * as fs from 'fs' async function importCatalog() { diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md index 240e81ae..8560b543 100644 --- a/docs/guides/inspection.md +++ b/docs/guides/inspection.md @@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI. ## Programmatic inspection ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '/data/brain' } diff --git a/docs/guides/installation.md b/docs/guides/installation.md index 0a36f632..20d40ea2 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -21,21 +21,21 @@ next: ## Install ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` Or with your preferred package manager: ```bash -bun add @soulcraft/brainy -yarn add @soulcraft/brainy -pnpm add @soulcraft/brainy +bun add @soulcraftlabs/brainy +yarn add @soulcraftlabs/brainy +pnpm add @soulcraftlabs/brainy ``` ## Verify ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -52,7 +52,7 @@ npm install @soulcraft/cor ``` ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ plugins: ['@soulcraft/cor'] }) await brain.init() // native providers registered during init @@ -71,7 +71,7 @@ remains available on npm if you need it. Brainy ships with full TypeScript types. No `@types/` package needed: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/migration-3.36.0.md b/docs/guides/migration-3.36.0.md index 8b1f239e..5f00534a 100644 --- a/docs/guides/migration-3.36.0.md +++ b/docs/guides/migration-3.36.0.md @@ -66,7 +66,7 @@ const results = await brain.search("query") **New diagnostics for capacity planning and performance tuning.** ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')} ### Step 1: Update Package ```bash -npm install @soulcraft/brainy@latest +npm install @soulcraftlabs/brainy@latest ``` ### Step 2: Restart Your Application @@ -134,7 +134,7 @@ npm run start ### Check Adaptive Sizing is Working ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -218,7 +218,7 @@ For debugging or compatibility testing: If you need to rollback to v3.35.0: ```bash -npm install @soulcraft/brainy@3.35.0 +npm install @soulcraftlabs/brainy@3.35.0 ``` **Note:** We don't anticipate any issues, but rollback is straightforward if needed. @@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) { ## Next Steps -1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest` +1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest` 2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements 3. 🎯 **Tune:** Adjust based on recommendations (if needed) 4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning diff --git a/docs/guides/model-loading.md b/docs/guides/model-loading.md index e5b7b1d6..cc1b2b6a 100644 --- a/docs/guides/model-loading.md +++ b/docs/guides/model-loading.md @@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings. ```bash # Bun as a runtime — supported and recommended -bun add @soulcraft/brainy +bun add @soulcraftlabs/brainy bun run server.ts ``` diff --git a/docs/guides/namespace-migration.md b/docs/guides/namespace-migration.md index fad3c766..f7d2c7f7 100644 --- a/docs/guides/namespace-migration.md +++ b/docs/guides/namespace-migration.md @@ -80,7 +80,7 @@ If you read raw stored records (fact-log scanners, export tooling), use the exported shape-aware splitters — they handle both record eras: ```typescript -import { splitNounMetadataRecord } from '@soulcraft/brainy' +import { splitNounMetadataRecord } from '@soulcraftlabs/brainy' const { reserved, custom } = splitNounMetadataRecord(rawRecord) // reserved = engine fields · custom = the user's bag, ANY names ``` @@ -88,7 +88,7 @@ const { reserved, custom } = splitNounMetadataRecord(rawRecord) Feature detection (never version-sniff): ```typescript -import * as brainy from '@soulcraft/brainy' +import * as brainy from '@soulcraftlabs/brainy' const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1' ``` diff --git a/docs/guides/nextjs-integration.md b/docs/guides/nextjs-integration.md index ab55e51f..25d6062d 100644 --- a/docs/guides/nextjs-integration.md +++ b/docs/guides/nextjs-integration.md @@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou ```bash npx create-next-app@latest my-brainy-app cd my-brainy-app -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Setup @@ -18,7 +18,7 @@ npm install @soulcraft/brainy // app/components/BrainyProvider.jsx 'use client' import { createContext, useContext, useEffect, useState } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const BrainyContext = createContext() @@ -271,7 +271,7 @@ export default function SearchPage() { ```javascript // app/api/search/route.js (App Router) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -332,7 +332,7 @@ export async function GET() { ```javascript // pages/api/search.js (Pages Router) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -374,7 +374,7 @@ export default async function handler(req, res) { ```javascript // app/api/data/route.js -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -418,7 +418,7 @@ export async function POST(request) { ```jsx // app/actions/brainy.js 'use server' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -630,7 +630,7 @@ CMD ["npm", "start"] /** @type {import('next').NextConfig} */ const nextConfig = { experimental: { - serverComponentsExternalPackages: ['@soulcraft/brainy'] + serverComponentsExternalPackages: ['@soulcraftlabs/brainy'] }, webpack: (config, { isServer }) => { if (!isServer) { @@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) { // app/contexts/BrainyContext.jsx 'use client' import { createContext, useContext, useReducer, useEffect } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const BrainyContext = createContext() @@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider' import { Search } from '../app/components/Search' // Mock Brainy -jest.mock('@soulcraft/brainy', () => ({ +jest.mock('@soulcraftlabs/brainy', () => ({ Brainy: jest.fn().mockImplementation(() => ({ init: jest.fn().mockResolvedValue(undefined), find: jest.fn().mockResolvedValue([ diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md index 268bc5fa..2984998b 100644 --- a/docs/guides/optimistic-concurrency.md +++ b/docs/guides/optimistic-concurrency.md @@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin Every distributed-job scheduler eventually wants this exact loop: ```ts -import { Brainy, RevisionConflictError } from '@soulcraft/brainy' +import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy' const LOCK_ID = '...uuid for this job slot...' @@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved: ```ts -import { GenerationConflictError } from '@soulcraft/brainy' +import { GenerationConflictError } from '@soulcraftlabs/brainy' async function addIfMissingByEmail(email: string, data: string) { for (let attempt = 0; attempt < 5; attempt++) { diff --git a/docs/guides/quick-start.md b/docs/guides/quick-start.md index 097c55fe..d9a4e896 100644 --- a/docs/guides/quick-start.md +++ b/docs/guides/quick-start.md @@ -18,13 +18,13 @@ Get Brainy running in under a minute. ## 1. Install ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ## 2. Initialize ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -67,7 +67,7 @@ await brain.relate({ ## 5. Query with Triple Intelligence ```typescript -import type { Result } from '@soulcraft/brainy' +import type { Result } from '@soulcraftlabs/brainy' // All three search paradigms in one call const results: Result[] = await brain.find({ diff --git a/docs/guides/standard-import-progress.md b/docs/guides/standard-import-progress.md index 9f2e2e5b..27dabe75 100644 --- a/docs/guides/standard-import-progress.md +++ b/docs/guides/standard-import-progress.md @@ -11,7 +11,7 @@ ### One Interface for Everything ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = await Brainy.create() @@ -78,7 +78,7 @@ interface ImportProgress { ```typescript import { useState } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' function UniversalImportProgress({ file }: { file: File }) { const [progress, setProgress] = useState({ @@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) { ```typescript import ora from 'ora' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function importWithProgress(filePath: string) { const spinner = ora('Starting import...').start() diff --git a/docs/guides/storage-adapters.md b/docs/guides/storage-adapters.md index 06ec9f3a..a4224bc8 100644 --- a/docs/guides/storage-adapters.md +++ b/docs/guides/storage-adapters.md @@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map). ## Quick start ```ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Filesystem (recommended for any persistent workload): const brain = new Brainy({ @@ -134,7 +134,7 @@ config; the `type` is optional. If you want to skip the factory: ```ts -import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy' +import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy' const fsStorage = new FileSystemStorage('./brainy-data') const memStorage = new MemoryStorage() diff --git a/docs/guides/subtypes-and-facets.md b/docs/guides/subtypes-and-facets.md index ff5de320..74311528 100644 --- a/docs/guides/subtypes-and-facets.md +++ b/docs/guides/subtypes-and-facets.md @@ -34,7 +34,7 @@ Three layers solve this: ### Write ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -240,7 +240,7 @@ await brain.migrateField({ A realistic adoption sequence for a brain that started without these primitives: ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } }) await brain.init() diff --git a/docs/guides/upgrading-7-to-8.md b/docs/guides/upgrading-7-to-8.md index a3c64fb9..53aa2a5c 100644 --- a/docs/guides/upgrading-7-to-8.md +++ b/docs/guides/upgrading-7-to-8.md @@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you. ## TL;DR -- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.** +- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.** If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**, with no operator action. - Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**. @@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and open it.** ```ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Opening the store is all that is required — recovery runs during init(). const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } }) @@ -182,5 +182,5 @@ and opening each store is sufficient. The recovery is copy-only, so no rollback of the recovery itself is ever needed. If you need to roll back the **whole** 7→8 upgrade, restore the directory from your pre-upgrade backup (retained automatically while recovery is incomplete, or -your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old +your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old branch layout in place, so a directory-level restore is the rollback path. diff --git a/docs/guides/vue-integration.md b/docs/guides/vue-integration.md index 7f7c6a06..34d18ebf 100644 --- a/docs/guides/vue-integration.md +++ b/docs/guides/vue-integration.md @@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N npm create vue@latest my-brainy-app cd my-brainy-app npm install -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Setup @@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun ```javascript // server/utils/brain.js (server-only — Nitro never bundles this into the client) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue' export default defineConfig({ plugins: [vue()], ssr: { - external: ['@soulcraft/brainy'] + external: ['@soulcraftlabs/brainy'] } }) ``` diff --git a/docs/neural-extraction.md b/docs/neural-extraction.md index 989b1b60..cfb6d764 100644 --- a/docs/neural-extraction.md +++ b/docs/neural-extraction.md @@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c ### Method 1: Brain Instance (Recommended) ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', { import { SmartExtractor, SmartRelationshipExtractor -} from '@soulcraft/brainy' +} from '@soulcraftlabs/brainy' // Or use subpath imports: -import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor' +import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor' const brain = new Brainy() await brain.init() @@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, { **Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration. ```typescript -import { SmartExtractor, FormatContext } from '@soulcraft/brainy' +import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy' const extractor = new SmartExtractor(brain, { minConfidence: 0.7, // Threshold @@ -229,7 +229,7 @@ interface ExtractionResult { **Relationship type classifier.** Determines verb/relationship types between entities. ```typescript -import { SmartRelationshipExtractor } from '@soulcraft/brainy' +import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' const relExtractor = new SmartRelationshipExtractor(brain, { minConfidence: 0.6, @@ -286,7 +286,7 @@ const rel = await relExtractor.infer( **Full extraction orchestrator.** Handles candidate detection, classification, and deduplication. ```typescript -import { NeuralEntityExtractor } from '@soulcraft/brainy' +import { NeuralEntityExtractor } from '@soulcraftlabs/brainy' const extractor = new NeuralEntityExtractor(brain) @@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location) ### Example 2: Excel Data Classification ```typescript -import { SmartExtractor } from '@soulcraft/brainy' +import { SmartExtractor } from '@soulcraftlabs/brainy' const extractor = new SmartExtractor(brain) @@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) { ### Example 3: Relationship Extraction ```typescript -import { SmartRelationshipExtractor } from '@soulcraft/brainy' +import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' const relExtractor = new SmartRelationshipExtractor(brain) diff --git a/docs/transactions.md b/docs/transactions.md index fce7d10e..cbea39c0 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing }) ### Basic Add Operation ```typescript -import { Brainy } from '@soulcraft/brainy' -import { NounType } from '@soulcraft/brainy/types' +import { Brainy } from '@soulcraftlabs/brainy' +import { NounType } from '@soulcraftlabs/brainy/types' const brain = new Brainy() await brain.init() @@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked ```typescript import { describe, it, expect } from 'vitest' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' describe('Transaction Tests', () => { it('should rollback on failure', async () => { diff --git a/docs/universal-display-augmentation.md b/docs/universal-display-augmentation.md index da42874c..464b91fb 100644 --- a/docs/universal-display-augmentation.md +++ b/docs/universal-display-augmentation.md @@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati ### Basic Usage ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brainy = new Brainy() await brainy.init() diff --git a/docs/vfs/PROJECTION_STRATEGY_API.md b/docs/vfs/PROJECTION_STRATEGY_API.md index 380862e1..f1319d5b 100644 --- a/docs/vfs/PROJECTION_STRATEGY_API.md +++ b/docs/vfs/PROJECTION_STRATEGY_API.md @@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low): ### Step 1: Create the Strategy Class ```typescript -import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' -import { Brainy } from '@soulcraft/brainy' -import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs' +import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' +import { Brainy } from '@soulcraftlabs/brainy' +import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs' export class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' @@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy { ### Step 2: Register the Strategy ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { PriorityProjection } from './PriorityProjection' const brain = new Brainy() @@ -537,7 +537,7 @@ Use the projection's resolve cache: ```typescript import { describe, it, expect, beforeAll } from 'vitest' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { PriorityProjection } from './PriorityProjection' describe('PriorityProjection', () => { @@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) { 3. Use appropriate limits: Don't fetch more than needed ### Type errors -1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'` +1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'` 2. Use `as VFSEntity` when mapping results 3. Check BaseProjectionStrategy import diff --git a/docs/vfs/QUICK_START.md b/docs/vfs/QUICK_START.md index 8b0efce6..4a1f83dc 100644 --- a/docs/vfs/QUICK_START.md +++ b/docs/vfs/QUICK_START.md @@ -14,11 +14,11 @@ A file explorer that: ## ⚡ Step 1: Basic Setup (1 minute) ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // ✅ CORRECT: Use filesystem storage for production const brain = new Brainy({ @@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns: ```tsx import React, { useState, useEffect } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' export function FileExplorer() { const [brain, setBrain] = useState(null) @@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next: ### "Module not found" errors ```bash # Make sure you're using the right import -npm ls @soulcraft/brainy # Check version -npm install @soulcraft/brainy@latest # Update if needed +npm ls @soulcraftlabs/brainy # Check version +npm install @soulcraftlabs/brainy@latest # Update if needed ``` ### "VFS not initialized" errors diff --git a/docs/vfs/README.md b/docs/vfs/README.md index b95f0d7b..a94910c9 100644 --- a/docs/vfs/README.md +++ b/docs/vfs/README.md @@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne ## Quick Start ```javascript -import { VirtualFileSystem } from '@soulcraft/brainy/vfs' +import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs' // Initialize the VFS const vfs = new VirtualFileSystem({ @@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system: ## Installation ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ## Requirements diff --git a/docs/vfs/ROADMAP.md b/docs/vfs/ROADMAP.md index 93c5b901..c8d15cd2 100644 --- a/docs/vfs/ROADMAP.md +++ b/docs/vfs/ROADMAP.md @@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows. ```typescript // Planned (research phase) -import { mountVFS } from '@soulcraft/brainy/vfs/fuse' +import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse' await mountVFS(vfs, { mountPoint: '/mnt/brainy', @@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested ### Express.js Static Middleware ```typescript // Wanted: Community contribution -import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express' +import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express' app.use('/files', createStaticMiddleware(vfs, { index: ['index.html', 'index.md'], @@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, { ### VSCode Extension ```typescript // Wanted: Community contribution -import { VFSProvider } from '@soulcraft/brainy/vfs/vscode' +import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode' const provider = new VFSProvider(vfs) vscode.workspace.registerFileSystemProvider('brainy', provider) diff --git a/docs/vfs/SEMANTIC_VFS.md b/docs/vfs/SEMANTIC_VFS.md index 9298c822..f34ee9ae 100644 --- a/docs/vfs/SEMANTIC_VFS.md +++ b/docs/vfs/SEMANTIC_VFS.md @@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true Create your own semantic dimensions: ```typescript -import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' +import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' diff --git a/docs/vfs/VFS_API_GUIDE.md b/docs/vfs/VFS_API_GUIDE.md index e0c6a94c..5dcaaeb8 100644 --- a/docs/vfs/VFS_API_GUIDE.md +++ b/docs/vfs/VFS_API_GUIDE.md @@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha ## Quick Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Initialize Brainy const brain = new Brainy({ @@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123') VFS uses standard POSIX-style errors: ```typescript -import { VFSError, VFSErrorCode } from '@soulcraft/brainy' +import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy' try { await vfs.readFile('/nonexistent.txt') diff --git a/docs/vfs/VFS_CORE.md b/docs/vfs/VFS_CORE.md index 1eeaf9f8..c1d502c0 100644 --- a/docs/vfs/VFS_CORE.md +++ b/docs/vfs/VFS_CORE.md @@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities: #### GitBridge Usage ```javascript // Import and instantiate GitBridge -import { GitBridge } from '@soulcraft/brainy' +import { GitBridge } from '@soulcraftlabs/brainy' const gitBridge = new GitBridge(vfs, brain) // Export VFS to Git repository structure @@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare ## Complete Example ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function vfsExample() { // Initialize diff --git a/docs/vfs/VFS_GRAPH_TYPES.md b/docs/vfs/VFS_GRAPH_TYPES.md index 3c1f30f0..478bef7f 100644 --- a/docs/vfs/VFS_GRAPH_TYPES.md +++ b/docs/vfs/VFS_GRAPH_TYPES.md @@ -196,5 +196,5 @@ await brain.relate({ Always import and use the type enums: ```javascript -import { NounType, VerbType } from '@soulcraft/brainy' +import { NounType, VerbType } from '@soulcraftlabs/brainy' ``` \ No newline at end of file diff --git a/docs/vfs/VFS_INITIALIZATION.md b/docs/vfs/VFS_INITIALIZATION.md index 97e6b0bf..fd12fc71 100644 --- a/docs/vfs/VFS_INITIALIZATION.md +++ b/docs/vfs/VFS_INITIALIZATION.md @@ -5,7 +5,7 @@ The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed! ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Create and initialize Brainy const brain = new Brainy({ @@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e ## Complete Example ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function useVFS() { // Initialize Brainy @@ -100,7 +100,7 @@ useVFS().catch(console.error) ## TypeScript Usage ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' class FileManager { private brain: Brainy diff --git a/docs/vfs/building-file-explorers.md b/docs/vfs/building-file-explorers.md index 6bb31871..7514c12e 100644 --- a/docs/vfs/building-file-explorers.md +++ b/docs/vfs/building-file-explorers.md @@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues: ### Method 1: Use `getDirectChildren()` (Recommended) ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -97,7 +97,7 @@ Here's a complete example using React: ```tsx import React, { useState, useEffect } from 'react' -import { VirtualFileSystem } from '@soulcraft/brainy' +import { VirtualFileSystem } from '@soulcraftlabs/brainy' interface FileNode { name: string @@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) { If you must build trees manually from flat lists, use the `VFSTreeUtils`: ```typescript -import { VFSTreeUtils } from '@soulcraft/brainy/vfs' +import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs' // Get all entities somehow const allEntities = await vfs.getDescendants('/root') diff --git a/examples/bluesky-distributed-setup.js b/examples/bluesky-distributed-setup.js index 9e83cf25..e3b33506 100644 --- a/examples/bluesky-distributed-setup.js +++ b/examples/bluesky-distributed-setup.js @@ -7,7 +7,7 @@ * the Bluesky firehose with Brainy's distributed architecture */ -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { WebSocket } from 'ws' // ===================================================== diff --git a/examples/monitor-cache-performance.ts b/examples/monitor-cache-performance.ts index 9d50d476..87c965a2 100644 --- a/examples/monitor-cache-performance.ts +++ b/examples/monitor-cache-performance.ts @@ -14,7 +14,7 @@ * ts-node examples/monitor-cache-performance.ts */ -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // ANSI color codes for pretty output const colors = { diff --git a/integrations/README.md b/integrations/README.md index aa3d795b..de156623 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu ## Quick Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature. ### Minimal (in-memory): ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions()) ```typescript import express from 'express' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = express() const brain = new Brainy({ @@ -232,7 +232,7 @@ app.listen(3000, () => { ```typescript import { Hono } from 'hono' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = new Hono() diff --git a/integrations/google-sheets/README.md b/integrations/google-sheets/README.md index b2b0af3a..8309a30a 100644 --- a/integrations/google-sheets/README.md +++ b/integrations/google-sheets/README.md @@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings. The simplest way to enable all integrations: ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -112,7 +112,7 @@ With Express: ```javascript import express from 'express' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = express() const brain = new Brainy({ integrations: true }) diff --git a/package-lock.json b/package-lock.json index fb88681f..9cc3c6b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "version": "10.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "version": "10.4.2", "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 17983c19..2312fcb0 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "version": "10.4.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", @@ -126,15 +126,16 @@ "license": "MIT", "private": false, "publishConfig": { - "access": "public" + "access": "public", + "registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" }, - "homepage": "https://source.soulcraft.com/soulcraft/brainy", + "homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy", "bugs": { - "url": "https://source.soulcraft.com/soulcraft/brainy/issues" + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues" }, "repository": { "type": "git", - "url": "git+https://source.soulcraft.com/soulcraft/brainy.git" + "url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git" }, "files": [ "dist/**/*.js", diff --git a/scripts/release.sh b/scripts/release.sh index 5d03e66b..be1d6d6b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -15,11 +15,11 @@ NC='\033[0m' # No Color RELEASE_TYPE="${1:-patch}" # patch, minor, or major SKIP_TESTS=false DRY_RUN=false -# --source-only: the HOME leg only — tag, CI's publish to The Source, and the -# release page; NO storefront (npmjs) publish, NO pair verification, NO docs -# push. The pair-gate shape: a prerelease the fleet's other engine devDeps -# from our own registry while the pair is proven, never a public artifact. -# Refused for a non-prerelease version — a public floor is always a pair. +# --source-only is now a no-op: The Source is the one registry, so every +# release already ships Source-only — tag, CI's publish to The Source, the +# release page, and the docs push, with no separate storefront leg to skip. +# The flag is still accepted (for backward-compatible invocations) and just +# prints a notice; it no longer changes behavior. SOURCE_ONLY=false for arg in "$@"; do @@ -109,7 +109,7 @@ else ;; *) echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}" - echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run] [--source-only (prereleases only)]" + echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run] [--source-only (no-op; The Source is the one registry)]" exit 1 ;; esac @@ -129,11 +129,7 @@ if [ "$PRERELEASE" = true ]; then echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}" fi if [ "$SOURCE_ONLY" = true ]; then - if [ "$PRERELEASE" != true ]; then - echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}" - exit 1 - fi - echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}" + echo -e "${YELLOW}⚠️ The Source is the one registry; --source-only is implied${NC}" fi echo "" @@ -209,9 +205,9 @@ echo -e "${GREEN}✅ Pushed to origin${NC}\n" # .forgejo/workflows/publish-source.yml, which builds and publishes on The # Source's own runner (datacenter-side: seconds, not the laptop's WAN timing # out on an 87MB tarball PUT). The laptop holds no home-registry publish -# credential anymore; it only waits for CI's result before trusting the -# home/npmjs pair enough to publish the storefront leg. -SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" +# credential anymore; it only waits for CI's result before continuing on to +# the release page and the docs push. +SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" SOURCE_POLL_INTERVAL_S=15 SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml # backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0); @@ -219,7 +215,7 @@ SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequen echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}" SOURCE_LANDED=false for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do - LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") + LANDED_VERSION=$(npm view "@soulcraftlabs/brainy@${NEW_VERSION}" version "--@soulcraftlabs:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then SOURCE_LANDED=true break @@ -232,57 +228,11 @@ if [ "$SOURCE_LANDED" = true ]; then echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n" else echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}" - echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}" - echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}" + echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraftlabs/brainy@${NEW_VERSION} never became visible on the${NC}" + echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting.${NC}" exit 1 fi -if [ "$SOURCE_ONLY" = true ]; then - echo -e "${YELLOW}9️⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n" -else - echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}" - # BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download - # the tarball The Source serves and publish that file, never a fresh local pack - # (a local rebuild can differ byte-wise, and the fleet verifies the pair by - # shasum across registries). - STOREFRONT_TMP="$(mktemp -d)" - (cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null) - SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)" - echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}" - npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" - rm -rf "$STOREFRONT_TMP" - # Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. - npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true - # Verify the pair is byte-identical by registry-reported shasum — divergence - # here means the storefront leg must be treated as failed, loudly. RETRIED - # with raw curl: npmjs metadata propagates with a lag measured in minutes, - # and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a - # raw curl of the registry document already confirmed byte-identity. The - # probe now reads the registry JSON directly (no npm cache in the path) and - # gives propagation up to 5 minutes before calling the pair divergent. - NPMJS_VERIFY_ATTEMPTS=20 - NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace - SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable") - PAIR_IDENTICAL=false - for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do - NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \ - | node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \ - || echo "") - if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then - PAIR_IDENTICAL=true - break - fi - echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}" - sleep "$NPMJS_VERIFY_INTERVAL_S" - done - if [ "$PAIR_IDENTICAL" = true ]; then - echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n" - else - echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n" - exit 1 - fi -fi - # Step 11: Release object on The Source (presentational — the tag, CHANGELOG, # and RELEASES.md are the record; this just gives The Source's UI a release page). echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" @@ -303,24 +253,15 @@ fi # DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — # that already happened) when a push errors, so the docs site never # silently trails npm. -if [ "$SOURCE_ONLY" = true ]; then - echo -e "${YELLOW}1️⃣2️⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n" +echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" +if node scripts/push-docs.js; then + echo -e "${GREEN}✅ Docs push step done${NC}\n" else - echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" - if node scripts/push-docs.js; then - echo -e "${GREEN}✅ Docs push step done${NC}\n" - else - echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" - fi + echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" fi echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" -if [ "$SOURCE_ONLY" = true ]; then - echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}" -else - echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}" -fi echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" diff --git a/src/brainy.ts b/src/brainy.ts index ed958a67..966d3188 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -952,14 +952,14 @@ export class Brainy implements BrainyInterface { * extends FileSystemStorage`) inherit new methods Brainy adds to * `FileSystemStorage` / `BaseStorage` automatically — `typeof` walks the * prototype chain, so there's no in-package version skew to worry about as - * long as the plugin's own dist resolves `@soulcraft/brainy` dynamically + * long as the plugin's own dist resolves `@soulcraftlabs/brainy` dynamically * (which Cortex 2.2.x onward does — see * `node_modules/@soulcraft/cor/dist/storage/mmapFileSystemStorage.js`). * * This helper exists for the **build/install** failure modes the import * resolution can't catch: * - Stale `node_modules` left over from a prior `bun install` against - * `@soulcraft/brainy ≤7.20.x`. + * `@soulcraftlabs/brainy ≤7.20.x`. * - Lockfile drift pinning brainy below the version that introduced the * method. * - Docker layer caches that reuse a `node_modules` from an earlier image. @@ -1175,7 +1175,7 @@ export class Brainy implements BrainyInterface { `and the flush-request RPC are disabled for this directory. ` + `Likely fix: clean install (\`rm -rf node_modules bun.lockb && ` + `bun install\`) or rebuild your container image to refresh ` + - `\`@soulcraft/brainy\` to ≥7.21. See docs/concepts/storage-adapters.md.` + `\`@soulcraftlabs/brainy\` to ≥7.21. See docs/concepts/storage-adapters.md.` ) } else { console.warn( diff --git a/src/db/errors.ts b/src/db/errors.ts index e20488f8..3b4c1af6 100644 --- a/src/db/errors.ts +++ b/src/db/errors.ts @@ -28,7 +28,7 @@ * speculative `with()` overlay; the canonical storage walk only ever answers * "what is live right now." * - * All are exported from the package root (`@soulcraft/brainy`). + * All are exported from the package root (`@soulcraftlabs/brainy`). */ /** diff --git a/src/embeddings/wasm/modelLoader.ts b/src/embeddings/wasm/modelLoader.ts index 45ffc4d3..b39d90ea 100644 --- a/src/embeddings/wasm/modelLoader.ts +++ b/src/embeddings/wasm/modelLoader.ts @@ -128,7 +128,7 @@ async function loadBunAssets(): Promise { } // Strategy 2: node_modules path relative to CWD (for installed packages) - const nmPath = './node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2' + const nmPath = './node_modules/@soulcraftlabs/brainy/assets/models/all-MiniLM-L6-v2' pathsToTry.push([ `${nmPath}/model.safetensors`, `${nmPath}/tokenizer.json`, @@ -168,9 +168,9 @@ async function loadBunAssets(): Promise { // If all strategies fail, provide helpful error message throw new Error( 'Could not load model assets. For bun --compile, ensure model files are accessible:\n' + - ' Option 1: Keep node_modules/@soulcraft/brainy/assets/ alongside your binary\n' + + ' Option 1: Keep node_modules/@soulcraftlabs/brainy/assets/ alongside your binary\n' + ' Option 2: Copy assets/ folder to your working directory\n' + - ' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraft/brainy/assets/**/*"' + ' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraftlabs/brainy/assets/**/*"' ) } @@ -190,7 +190,7 @@ async function loadNodeAssets(): Promise { if (!fs.existsSync(assetsDir)) { throw new Error( `Model assets not found: ${assetsDir}\n` + - `Ensure @soulcraft/brainy is installed correctly.` + `Ensure @soulcraftlabs/brainy is installed correctly.` ) } diff --git a/src/errors/notFound.ts b/src/errors/notFound.ts index 8eca9b2e..628797a6 100644 --- a/src/errors/notFound.ts +++ b/src/errors/notFound.ts @@ -14,7 +14,7 @@ * - {@link RelationNotFoundError} — a referenced relationship (verb) does * not exist. * - * Both are exported from the package root (`@soulcraft/brainy`). + * Both are exported from the package root (`@soulcraftlabs/brainy`). */ /** diff --git a/src/integrations/index.ts b/src/integrations/index.ts index 757a9fe5..6a6734d9 100644 --- a/src/integrations/index.ts +++ b/src/integrations/index.ts @@ -9,7 +9,7 @@ * * @example Enable integrations (recommended) * ```typescript - * import { Brainy } from '@soulcraft/brainy' + * import { Brainy } from '@soulcraftlabs/brainy' * * const brain = new Brainy({ integrations: true }) * await brain.init() diff --git a/src/mcp/README.md b/src/mcp/README.md index c69a3b24..092534a1 100644 --- a/src/mcp/README.md +++ b/src/mcp/README.md @@ -41,7 +41,7 @@ The `BrainyMCPService` has been refactored to separate the core functionality fr ### In Any Environment (Browser, Node.js, Server) ```typescript -import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy' +import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraftlabs/brainy' // Create a Brainy instance const brainyData = new Brainy() @@ -81,7 +81,7 @@ const toolResponse = await toolset.handleRequest({ ### In Browser Environment (Core Functionality Only) ```typescript -import { Brainy, BrainyMCPService } from '@soulcraft/brainy' +import { Brainy, BrainyMCPService } from '@soulcraftlabs/brainy' // Create a Brainy instance const brainyData = new Brainy() diff --git a/src/plugin.ts b/src/plugin.ts index bfdc403a..b1aef8e0 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -22,7 +22,7 @@ import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js' // Re-export the provider contracts that already live closer to their // implementations so a plugin author (Cor) can import the *entire* -// provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`. +// provider surface from one stable entrypoint: `@soulcraftlabs/brainy/plugin`. export type { ColumnStoreProvider } from './indexes/columnStore/types.js' export type { AggregationProvider, @@ -41,7 +41,7 @@ export interface BrainyPlugin { name: string /** - * Optional semver range of `@soulcraft/brainy` this plugin supports + * Optional semver range of `@soulcraftlabs/brainy` this plugin supports * (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is * OUTSIDE the range, `init()` THROWS rather than silently falling back to the * default JS engine. This is the version-coupling guard for the native diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 63356828..d9934c3b 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -215,7 +215,7 @@ export interface ScoreExplanation { * * @example * ```ts - * declare module '@soulcraft/brainy' { + * declare module '@soulcraftlabs/brainy' { * interface SubtypeRegistry { * // For NounType.Person, subtype 'employee': * 'person:employee': { employeeId: string; department: string } diff --git a/src/types/reservedFields.ts b/src/types/reservedFields.ts index 15b585c5..ce2108f8 100644 --- a/src/types/reservedFields.ts +++ b/src/types/reservedFields.ts @@ -65,7 +65,7 @@ * | `_rev` | system-managed revision counter — pass `ifRev` to `update()` for CAS | * * @example - * import { RESERVED_ENTITY_FIELDS } from '@soulcraft/brainy' + * import { RESERVED_ENTITY_FIELDS } from '@soulcraftlabs/brainy' * const isReserved = (key: string) => * (RESERVED_ENTITY_FIELDS as readonly string[]).includes(key) */ diff --git a/src/utils/brainyTypes.ts b/src/utils/brainyTypes.ts index 7db469bb..a008a5e6 100644 --- a/src/utils/brainyTypes.ts +++ b/src/utils/brainyTypes.ts @@ -6,7 +6,7 @@ * * @example * ```typescript - * import { BrainyTypes } from '@soulcraft/brainy' + * import { BrainyTypes } from '@soulcraftlabs/brainy' * * // Get all available types * const nounTypes = BrainyTypes.nouns // ['Person', 'Organization', ...] diff --git a/src/utils/version.ts b/src/utils/version.ts index d616cee3..f302eae0 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -1,6 +1,6 @@ /** * @module utils/version - * @description Resolves the running `@soulcraft/brainy` package version. Brainy 8.0 + * @description Resolves the running `@soulcraftlabs/brainy` package version. Brainy 8.0 * targets Node-like runtimes only (Node.js, Bun, Deno — all expose `node:fs`), so the * version is read **synchronously** from `package.json` on first call and cached. * diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index 31b9b216..b5817c3d 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -15,7 +15,7 @@ * - Hook 2: the public `brain.stampBrainFormat()` the provider calls once its * background migration has verified-and-swapped, authoring the shared * `_system/brain-format.json` marker. - * - Hook 3: the marker module is re-exported at `@soulcraft/brainy/brain-format` + * - Hook 3: the marker module is re-exported at `@soulcraftlabs/brainy/brain-format` * so cor reads the SAME `EXPECTED_INDEX_EPOCH` / `CURRENT_DATA_FORMAT` constants * (single source of truth, no duplicated value). * @@ -242,7 +242,7 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b // --- Hook 3: marker module export ---------------------------------------- it('the brain-format marker module exports the compiled epoch + data-format constants', () => { - // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both + // cor imports these from '@soulcraftlabs/brainy/brain-format' (Hook 3) so both // sides share ONE source of truth — no duplicated constant to drift. // Epoch 3: the namespace-law key split (bare user keys · literal // 'system.' scalars, 2026-08-03) — every brain rebuilds onto the From 384f4b6b9c908baf1e6319f56427be1b50373353 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 17:10:30 -0700 Subject: [PATCH 08/42] chore(release): 10.4.3 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d13a2d66..56757f1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2...v10.4.3) (2026-08-27) + +- Merge branch 'next/open-brainy-rename' (a58372f0) +- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83) +- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24) + + ### [10.4.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) - docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) diff --git a/package-lock.json b/package-lock.json index 9cc3c6b0..4d247780 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.2", + "version": "10.4.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.2", + "version": "10.4.3", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 2312fcb0..bb6b5a47 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.2", + "version": "10.4.3", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 38c3397b600e776fcd737445a996a6cc37d2f315 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 17:26:44 -0700 Subject: [PATCH 09/42] =?UTF-8?q?docs:=20repository=20links=20point=20at?= =?UTF-8?q?=20soulcraftlabs/open-brainy=20=E2=80=94=20the=20soulcraft/brai?= =?UTF-8?q?ny=20path=20becomes=20the=20native=20engine's=20repo=20tonight?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 38 +++++++++++++++++++------------------- CONTRIBUTING.md | 4 ++-- README.md | 2 +- RELEASES.md | 2 +- scripts/release.sh | 4 ++-- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56757f1c..c7790837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,19 +2,19 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.4.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2...v10.4.3) (2026-08-27) +### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27) - Merge branch 'next/open-brainy-rename' (a58372f0) - chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83) - docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24) -### [10.4.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) +### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) - docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) -### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) +### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) - Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) - fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) @@ -31,23 +31,23 @@ All notable changes to this project will be documented in this file. See [standa - chore(release): 10.4.1-rc.1 (7870dc40) -### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26) +### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) - docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) -### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) +### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) - docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) -### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) +### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) - feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) -### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) +### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) - fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) - Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) @@ -56,7 +56,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) -### [10.4.0-rc.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) +### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) - test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) - feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) @@ -67,7 +67,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) -### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) +### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) - ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) - chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) @@ -80,13 +80,13 @@ All notable changes to this project will be documented in this file. See [standa - ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) -### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18) +### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18) - docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) - fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) -### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18) +### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18) - docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) - fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) @@ -95,14 +95,14 @@ All notable changes to this project will be documented in this file. See [standa - feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) -### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17) +### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17) - docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) - ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) - fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) -### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13) +### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13) - docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) - fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) @@ -111,7 +111,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) -### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12) +### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12) - fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) - fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) @@ -143,7 +143,7 @@ All notable changes to this project will be documented in this file. See [standa - test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) -### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04) +### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04) - docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) - fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed) @@ -178,7 +178,7 @@ All notable changes to this project will be documented in this file. See [standa - feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b) -### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27) +### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27) - docs: the last two archived-host links point home (91ef1c8b) - feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) @@ -187,19 +187,19 @@ All notable changes to this project will be documented in this file. See [standa - ci: run the pipeline on the forge (999d0ebb) -### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03) +### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03) - docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) - fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) -### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) +### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29) - docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) - fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) -### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) +### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) - fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d277091d..50860cb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ may find elsewhere in the repo's history. ## Where the project lives -The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**. +The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**. It's anonymously readable and cloneable — no account needed to browse, clone, or build. @@ -31,7 +31,7 @@ fine) to talk through the approach saves everyone rework. ## Development setup ```bash -git clone https://source.soulcraft.com/soulcraft/brainy.git +git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git cd brainy npm install npm run build diff --git a/README.md b/README.md index 47a9123a..762c9ec3 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

Package on The Source Repository - CI + CI Documentation MIT License TypeScript diff --git a/RELEASES.md b/RELEASES.md index b89beba0..4d716efa 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,7 +1,7 @@ # @soulcraft/brainy — Release Notes for Consumers This file is the **quick reference for downstream sessions** tracking Brainy changes. -Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases +Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases **How to use:** Brainy is the underlying data engine for downstream applications. Read this when: - Upgrading `@soulcraft/brainy` in your application diff --git a/scripts/release.sh b/scripts/release.sh index be1d6d6b..08293e3a 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,7 +154,7 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) ${COMMITS} " @@ -264,4 +264,4 @@ echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━ echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" -echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" +echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${NEW_VERSION}${NC}" From e652162c1fe2e86cbdd0441094938e4717f10829 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:17:20 -0700 Subject: [PATCH 10/42] fix(storage): a clean close is recorded, and the writer lock is always given up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production restart made this necessary: a service stopped with exit code 0, having awaited close() on every pooled brain, and its next boot announced "Overwriting stale writer lock ... appears dead" for every store it owned. Nothing had crashed. "The recorded pid is gone" is equally true of an orderly restart and of a crash, so the verdict could not tell an operator which one they had — and when the OS recycles a pid it fails the other way, refusing to open a store whose writer died days ago. Three changes, all at the law: - close() is two parts, and the second is unconditional. The durable steps (flush, markers, component close, plugin deactivate, buffer drain) move to closeDurableSteps(); the terminal releases — the flush-request watcher, the WRITER LOCK, the VFS timers, the terminal `closed` flag — always run. The original failure is narrated with what it costs the next open, then rethrown. - releaseWriterLock() writes a CLEAN-CLOSE RECORD (`locks/_writer.close`) naming the lock generation it released; the next claim consumes it, so a record can never vouch for a later crash. An open reads the record instead of guessing: recorded → nothing to recover; absent → say so, and name the crash recovery this open will now run. - The signal path stops failing in a batch. It was one try around a loop over every open brain, so the first instance whose flush rejected stranded every remaining brain's lock and markers — at exit code 0. Now: per-instance isolation, the generation store's close (the clean-shutdown marker, without which the next open folds the whole log) is part of shutdown, the lock is given up in a finally, and the handler no longer calls process.exit() when the host application has its own signal handler — that race truncated the host's own close() mid-flight. Pins: tests/integration/writer-lock-clean-close.test.ts — completed close leaves no lock and a consumed-once record with a silent reopen; a failing durable step still releases and still rethrows; SIGKILL leaves the lock with no record and the reopen names the crash; a host SIGTERM handler runs to completion. Branch plan (10 lines): 1. writer lock: clean-close record + always-release close [this commit] 2. open narration: an always-on channel; production clamps prodLog to ERROR, which is why a three-minute open printed nothing 3. open narration: per-phase lines as each phase ENDS, with progress cadence 4. measure both real-store fixtures on the box, before/after 5. move the generation-log fold out of the foreground where the serving law allows; durable resumable progress marker 6. same for the VFS bootstrap 7. counts: a legacy container-rule ledger must not keep serving wrong denominators; counts.json written atomically 8. counts pin with scar directories; two copies of one archive agree 9. docs/canonical-layout-ratification.md — 12 facts confirmed/corrected 10. report: MEASURED before/after, findings, and whether this is 10.4.4 --- src/brainy.ts | 322 ++++++++++++------ src/storage/adapters/fileSystemStorage.ts | 147 +++++++- src/storage/baseStorage.ts | 30 ++ .../writer-lock-clean-close.test.ts | 250 ++++++++++++++ 4 files changed, 641 insertions(+), 108 deletions(-) create mode 100644 tests/integration/writer-lock-clean-close.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 966d3188..957920f5 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1856,76 +1856,112 @@ export class Brainy implements BrainyInterface { * NOTE: Registers globally (once for all instances) to avoid MaxListenersExceededWarning */ private registerShutdownHooks(): void { + /** + * The signal-path shutdown. THREE LAWS, each written by a production + * shutdown that looked clean and wasn't: + * + * 1. PER-INSTANCE ISOLATION. This used to be one `try` around a loop over + * every open brain: the first instance whose flush rejected aborted the + * loop, so every remaining brain kept its writer lock and its unwritten + * markers — and the process still exited 0. A pool of brains failed in + * a batch, not one at a time. + * 2. THE MARKER IS PART OF SHUTDOWN. Flushing the indexes without closing + * the generation store leaves the clean-shutdown marker unwritten, so + * the NEXT open reads the store as crashed and folds the whole + * generation log — measured in tens of seconds on a real store, paid on + * every restart, after a shutdown the operator saw exit 0. + * 3. THE LOCK IS ALWAYS GIVEN UP. In a `finally`, per instance: a process + * on its way out holds nothing. + */ const flushOnShutdown = async () => { console.log('Shutdown signal received - flushing pending data...') - try { - let flushedCount = 0 - for (const instance of Brainy.instances) { - if (instance.initialized) { - // Flush all buffered data, then close to release resources (timers, handles) - await Promise.all([ - (async () => { - if (instance.storage && typeof instance.storage.flushCounts === 'function') { - await instance.storage.flushCounts() - } - })(), - (async () => { - if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { - await instance.metadataIndex.flush() - } - })(), - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { - await instance.graphIndex.flush() - } - })(), - (async () => { - if (instance.index && typeof instance.index.flush === 'function') { - await instance.index.flush() - } - })() - ]) - // Close components to stop timers that would prevent clean process exit - await Promise.all([ - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { - await instance.graphIndex.close() - } - })(), - (async () => { - const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks - if (index && typeof index.close === 'function') { - await index.close() - } - })(), - (async () => { - const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks - if (metadataIndex && typeof metadataIndex.close === 'function') { - await metadataIndex.close() - } - })(), - // Release the writer lock so a successor process can take over. - // No-op for readers and for backends without locking. - (async () => { - if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { - await instance.storage.releaseWriterLock() - } - })(), - // Stop the flush-request watcher to release its interval timer. - (async () => { - if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { - instance.storage.stopFlushRequestWatcher() - } - })(), - ]) - flushedCount++ + let flushedCount = 0 + let failedCount = 0 + // Snapshot: close() splices Brainy.instances while we iterate. + for (const instance of [...Brainy.instances]) { + if (!instance.initialized) continue + try { + // Flush all buffered data (parallel across components, this brain only). + await Promise.all([ + (async () => { + if (instance.storage && typeof instance.storage.flushCounts === 'function') { + await instance.storage.flushCounts() + } + })(), + (async () => { + if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { + await instance.metadataIndex.flush() + } + })(), + (async () => { + if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { + await instance.graphIndex.flush() + } + })(), + (async () => { + if (instance.index && typeof instance.index.flush === 'function') { + await instance.index.flush() + } + })() + ]) + + // Close the generation store: persists the counter, advances the + // fold checkpoint, and stamps the clean-shutdown marker LAST — the + // one step that decides whether the next open adopts or folds. Law 2. + if (instance.generationStore && !instance.isReadOnly) { + await instance.generationStore.close() + } + + // Close components to stop timers that would prevent clean process exit + await Promise.all([ + (async () => { + if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { + await instance.graphIndex.close() + } + })(), + (async () => { + const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks + if (index && typeof index.close === 'function') { + await index.close() + } + })(), + (async () => { + const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks + if (metadataIndex && typeof metadataIndex.close === 'function') { + await metadataIndex.close() + } + })() + ]) + flushedCount++ + } catch (error) { + failedCount++ + console.error('Failed to flush one Brainy instance on shutdown:', error) + } finally { + // Law 3 — the lock and the watcher go regardless. + try { + if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { + instance.storage.stopFlushRequestWatcher() + } + } catch (error) { + console.error('Failed to stop the flush-request watcher on shutdown:', error) + } + try { + if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { + await instance.storage.releaseWriterLock() + } + } catch (error) { + console.error('Failed to release the writer lock on shutdown:', error) } } - if (flushedCount > 0) { - console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) - } - } catch (error) { - console.error('Failed to flush on shutdown:', error) + } + if (flushedCount > 0) { + console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) + } + if (failedCount > 0) { + console.error( + `${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` + + `their writer locks were released, but their next open will run crash recovery.` + ) } } @@ -1933,13 +1969,32 @@ export class Brainy implements BrainyInterface { // kept as statics so the last live instance's close() can deregister them // — the signal handles they hold are ref'd and would otherwise keep the // process alive forever after every brain is closed. + /** + * Exit the process ONLY when Brainy is the sole handler for this signal. + * + * Registering a signal listener suppresses Node's default terminate + * behaviour, so a library that attaches one must either exit or be sure + * someone else will. Brainy attaching one AND exiting was the wrong half + * of that choice for every host application with its own graceful + * shutdown: both handlers run concurrently, and whichever finishes first + * wins — a library flush finishing before an application's close() + * terminated that close mid-flight, at exit code 0, with locks and + * markers unwritten. When the host has its own handler (listener count + * above our own), the host owns the exit; Brainy only makes its data + * durable and steps aside. + */ + const exitIfSoleShutdownOwner = (signal: 'SIGTERM' | 'SIGINT'): void => { + if (process.listenerCount(signal) <= 1) { + process.exit(0) + } + } Brainy.sigtermListener = async () => { await flushOnShutdown() - process.exit(0) + exitIfSoleShutdownOwner('SIGTERM') } Brainy.sigintListener = async () => { await flushOnShutdown() - process.exit(0) + exitIfSoleShutdownOwner('SIGINT') } Brainy.beforeExitListener = async () => { // Self-deregister FIRST: Node re-emits 'beforeExit' after every event- @@ -18877,12 +18932,105 @@ export class Brainy implements BrainyInterface { } /** - * Close and cleanup + * @description Close and clean up: flush every buffered component, stamp + * the durability markers, release resources, then give up the writer lock. * - * Now flushes HNSW dirty nodes before closing - * This ensures deferred persistence mode data is saved + * TWO PARTS, AND THE SECOND IS UNCONDITIONAL. Everything that persists data + * runs in {@link closeDurableSteps}; the terminal releases — the flush-request + * watcher, the WRITER LOCK, the VFS timers, and the terminal `closed` flag — + * run whether those steps succeeded or not, in a `finally`. A close that + * threw halfway used to strand the writer lock on disk with this process's + * (soon dead) pid in it, so the next boot of every affected store announced + * `Overwriting stale writer lock … appears dead` after an orderly exit and + * an operator had to decide whether their database had crashed. A closed + * brain holds no lock — there is no failure for which the opposite is the + * safer answer. + * + * The original failure is never swallowed: it is narrated with what it costs + * the next open, then rethrown to the caller. + * @returns Nothing. + * @throws The first failure from the durable close steps, after the + * terminal releases have run. */ async close(): Promise { + let closeFailure: unknown = null + try { + await this.closeDurableSteps() + } catch (error) { + closeFailure = error + } + + // ---- TERMINAL RELEASES: always, even after a failure above ---- + + // Stop the cross-process flush-request watcher (no-op if never started). + try { + if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { + this.storage.stopFlushRequestWatcher() + } + } catch (error) { + console.warn('[Brainy] close: stopping the flush-request watcher failed:', error) + } + + // Release the writer lock. Runs after the metadata buffer drain in + // closeDurableSteps() — otherwise a pending write could land after a + // successor writer claimed the lock — and runs even if that drain threw: + // holding a lock from a process that is about to exit locks the store's + // next boot out of a clean verdict. + try { + if (this.storage && typeof this.storage.releaseWriterLock === 'function') { + await this.storage.releaseWriterLock() + } + } catch (error) { + console.warn('[Brainy] close: releasing the writer lock failed:', error) + } + + // Shut down the VFS: stops its background maintenance interval and the + // PathResolver's — both are ref'd timers that would keep the process + // alive after the last brain closes (consumer-reported hang). + try { + if (this._vfs) { + await this._vfs.close() + } + } catch (error) { + console.warn('[Brainy] close: VFS shutdown failed:', error) + } + + this.initialized = false + // close() is terminal: block lazy re-initialization on any subsequent + // operation (ensureInitialized() throws once this is set). Set even when + // the durable steps failed — a half-closed brain must not keep serving. + this.closed = true + + // Drop this instance from the global registry, and when it was the last + // one, deregister the global shutdown hooks — their ref'd signal handles + // would otherwise keep the process alive after every brain is closed. + const instanceIndex = Brainy.instances.indexOf(this) + if (instanceIndex !== -1) { + Brainy.instances.splice(instanceIndex, 1) + } + Brainy.deregisterShutdownHooksIfIdle() + + if (closeFailure !== null) { + console.error( + `[Brainy] close FAILED partway: ` + + `${closeFailure instanceof Error ? closeFailure.message : String(closeFailure)}\n` + + ` This brain is closed and holds no writer lock, but the clean-shutdown ` + + `marker may not have been written — the next open will run crash recovery ` + + `(a generation-log fold) and report its wall.` + ) + throw closeFailure + } + } + + /** + * @description The durable half of {@link close}: flush every component, + * persist the generation counter and its markers, close the components, + * deactivate plugins, drain the metadata write buffer. Separated from + * `close()` so the terminal releases there can run in a `finally` — see that + * method's contract. + * @returns Nothing. + */ + private async closeDurableSteps(): Promise { // Persistence cadence teardown: no background flush may fire after close // begins (close() runs its own final flush). if (this._persistIdleTimer) { @@ -19033,38 +19181,6 @@ export class Brainy implements BrainyInterface { } } - // Stop the cross-process flush-request watcher (no-op if never started). - if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { - this.storage.stopFlushRequestWatcher() - } - - // Release the writer lock (no-op for readers and for backends that don't - // hold a lock). Must run after the metadata buffer drain — otherwise a - // pending write could land after a successor writer claimed the lock. - if (this.storage && typeof this.storage.releaseWriterLock === 'function') { - await this.storage.releaseWriterLock() - } - - // Shut down the VFS: stops its background maintenance interval and the - // PathResolver's — both are ref'd timers that would keep the process - // alive after the last brain closes (consumer-reported hang). - if (this._vfs) { - await this._vfs.close() - } - - this.initialized = false - // close() is terminal: block lazy re-initialization on any subsequent - // operation (ensureInitialized() throws once this is set). - this.closed = true - - // Drop this instance from the global registry, and when it was the last - // one, deregister the global shutdown hooks — their ref'd signal handles - // would otherwise keep the process alive after every brain is closed. - const instanceIndex = Brainy.instances.indexOf(this) - if (instanceIndex !== -1) { - Brainy.instances.splice(instanceIndex, 1) - } - Brainy.deregisterShutdownHooksIfIdle() } } diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 4f2a43b0..a01c2015 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -14,7 +14,8 @@ import { StorageBatchConfig, SYSTEM_DIR, STATISTICS_KEY, - WriterLockInfo + WriterLockInfo, + WriterCloseRecord } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' @@ -99,6 +100,13 @@ export class FileSystemStorage extends BaseStorage { // timer rewrites the lock every 10s so stale-lock detection can tell a dead // writer from a slow one. The constant name matches the file path used. private static readonly WRITER_LOCK_FILE = '_writer.lock' + /** + * The clean-close record at `locks/_writer.close` (see + * {@link WriterCloseRecord}). Written when the lock is released, consumed by + * the next claim, so an open can distinguish "the previous writer left" from + * "the previous writer died" without inferring either from a pid. + */ + private static readonly WRITER_CLOSE_FILE = '_writer.close' private static readonly WRITER_HEARTBEAT_MS = 10_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout @@ -1902,11 +1910,24 @@ export class FileSystemStorage extends BaseStorage { rootDir: this.rootDir } await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) + await this.clearWriterCloseRecord() this.installWriterLock(info) return info } - const stale = !options?.force && (await this.isWriterLockStale(existing)) + // THE CLEAN-CLOSE RECORD IS CONSULTED FIRST (see WriterCloseRecord). + // A lock file whose release was RECORDED is bookkeeping left behind by + // an orderly shutdown, not evidence of a crash — take it over calmly + // and say so. Only when no record vouches for this lock do we fall + // back to inferring liveness from the pid, and then we say THAT + // honestly too: an unrecorded lock means the writer did not complete + // its close, so the store was not closed cleanly and this open pays + // recovery. + const closeRecord = await this.readWriterCloseRecord() + const releasedCleanly = + closeRecord !== null && this.closeRecordVouchesFor(closeRecord, existing) + const stale = + releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing))) if (!options?.force && !stale) { // Consumer-facing error contract: callers detect this case via // err.code and read the holder's details from err.lockInfo. @@ -1917,8 +1938,16 @@ export class FileSystemStorage extends BaseStorage { options?.force ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + `(was held by PID ${existing.pid} on ${existing.hostname}).` - : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + - `(PID ${existing.pid} on ${existing.hostname} appears dead).` + : releasedCleanly + ? `[brainy] Clearing the leftover writer lock for ${this.rootDir} — ` + + `PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` + + `${closeRecord!.closedAt} but could not remove the file. ` + + `Nothing to recover.` + : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + + `(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` + + `clean-close record — that writer did not finish closing, so this ` + + `store was not closed cleanly; open will run crash recovery and ` + + `report its wall).` ) // Takeover: verify the file still holds the lock we judged (a live // successor may have claimed meanwhile), then remove it and fall @@ -1972,6 +2001,12 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.unlink(claimTmp).catch(() => {}) } + // CONSUME the previous writer's clean-close record. It described the + // lock generation that just ended; leaving it in place would let it + // vouch for OUR lock if this process later dies without closing — + // turning a real crash into a "closed cleanly" verdict. One unlink. + await this.clearWriterCloseRecord() + this.installWriterLock(info) return info } @@ -2095,13 +2130,27 @@ export class FileSystemStorage extends BaseStorage { return } const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) + const released = this.writerLockInfo try { // Only delete if we still own it — avoid clobbering a successor that // claimed the lock via force-override. const current = await this.readWriterLock() - if (current && current.pid === this.writerLockInfo.pid && current.hostname === this.writerLockInfo.hostname) { + const ours = + current === null || + (current.pid === released.pid && current.hostname === released.hostname) + if (current && ours) { await fs.promises.unlink(lockFile) } + // THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this + // instance gives up a lock nobody else has taken — the unlink above + // having succeeded OR the file already being gone. The next open reads + // it instead of guessing from pid liveness: a recorded release is an + // orderly shutdown, an absent record is a writer that never finished + // closing. Not written when a successor holds the lock: our release is + // then a no-op and a record would slander their live lock. + if (ours) { + await this.writeWriterCloseRecord(released) + } } catch (err: any) { if (err.code !== 'ENOENT') { console.warn('[brainy] Failed to release writer lock file:', err) @@ -2111,6 +2160,94 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * @description Read the clean-close record at `locks/_writer.close`, or + * `null` when it is absent or unparseable. A torn record is treated as + * absent — the conservative direction, since an unreadable record can + * vouch for nothing. + * @returns The record, or null. + */ + public async readWriterCloseRecord(): Promise { + await this.ensureInitialized() + const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) + try { + const raw = await fs.promises.readFile(recordFile, 'utf-8') + const parsed = JSON.parse(raw) as WriterCloseRecord + if ( + typeof parsed?.pid !== 'number' || + typeof parsed?.hostname !== 'string' || + typeof parsed?.startedAt !== 'string' || + typeof parsed?.closedAt !== 'string' + ) { + return null + } + return parsed + } catch (err: any) { + if (err.code === 'ENOENT') return null + return null + } + } + + /** + * @description Whether a clean-close record describes the very lock + * generation `lock` represents. The match is pid + hostname + `startedAt`: + * `startedAt` is the lock generation's identity, so a record can never + * vouch for a LATER lock taken by the same pid on the same host (the + * same-process re-open path mints a fresh `startedAt`). + * @param record - The clean-close record read from disk. + * @param lock - The lock file's contents. + */ + private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean { + return ( + record.pid === lock.pid && + record.hostname === lock.hostname && + record.startedAt === lock.startedAt + ) + } + + /** + * @description Write the clean-close record for a lock this instance just + * released. Atomic (temp + rename) so a concurrent opener never reads half + * a record. A failure here costs the next open nothing but the honest + * fallback (pid liveness), so it warns rather than failing the close. + * @param released - The lock info this instance held. + */ + private async writeWriterCloseRecord(released: WriterLockInfo): Promise { + const record: WriterCloseRecord = { + pid: released.pid, + hostname: released.hostname, + startedAt: released.startedAt, + closedAt: new Date().toISOString(), + version: released.version + } + const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) + try { + await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2)) + } catch (err) { + console.warn( + `[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` + + `the next open will fall back to pid liveness and may report this orderly ` + + `shutdown as a crash:`, + err + ) + } + } + + /** + * @description Remove the clean-close record. Called by every successful + * lock claim so a record never outlives the lock generation it describes. + */ + private async clearWriterCloseRecord(): Promise { + const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) + try { + await fs.promises.unlink(recordFile) + } catch (err: any) { + if (err.code !== 'ENOENT') { + console.warn('[brainy] Failed to clear the writer clean-close record:', err) + } + } + } + public override async readWriterLock(): Promise { await this.ensureInitialized() const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 5510f93b..f518e68f 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -125,6 +125,36 @@ export interface WriterLockInfo { rootDir?: string // Convenience for log lines / error messages } +/** + * THE CLEAN-CLOSE RECORD. Written by `releaseWriterLock()` at the instant it + * gives up the writer lock, naming the lock identity it released. The next + * `acquireWriterLock()` reads it and can then say — from a RECORD, not from a + * guess — whether the previous writer left on purpose. + * + * Why a record and not PID liveness: "the recorded PID is no longer alive" is + * true of every orderly restart AND of every crash, so the two were reported + * identically ("appears dead") and neither could be trusted. Worse, the same + * inference fails the other way when the operating system RECYCLES the pid — + * a live unrelated process makes a long-dead writer's lock look held, and the + * store refuses to open naming a pid that was never Brainy. A record settles + * both: matched → the previous writer closed cleanly, nothing to recover; + * absent → say so, and name what recovery the open will now run. + * + * Lifecycle: written at release, consumed (deleted) by the next successful + * lock claim — a record must never outlive the lock generation it describes, + * or it would vouch for a later crash. + */ +export interface WriterCloseRecord { + pid: number + hostname: string + /** `startedAt` of the lock this close released — the identity match key. */ + startedAt: string + /** ISO timestamp at which the lock was released. */ + closedAt: string + /** Brainy version that performed the close. */ + version: string +} + /** * FNV-1a hash returning a 2-char hex bucket (00-ff). * Distributes system keys across 256 sub-prefixes to avoid diff --git a/tests/integration/writer-lock-clean-close.test.ts b/tests/integration/writer-lock-clean-close.test.ts new file mode 100644 index 00000000..7d9c59d6 --- /dev/null +++ b/tests/integration/writer-lock-clean-close.test.ts @@ -0,0 +1,250 @@ +/** + * @module tests/integration/writer-lock-clean-close + * @description THE CLEAN-CLOSE CONTRACT for the writer lock. + * + * A production restart made this lane necessary: a service stopped with exit + * code 0, having awaited `close()` on every pooled brain, and its next boot + * announced `[brainy] Overwriting stale writer lock … appears dead` for every + * store it owned. "The pid is gone" is equally true of an orderly restart and + * of a crash, so the message could not tell an operator which one they had. + * + * The contract pinned here: + * 1. A completed close leaves NO lock file and DOES leave a clean-close + * record; the next open says nothing about staleness. + * 2. The next lock claim CONSUMES that record — it may never outlive the + * lock generation it describes, or a later crash would read as clean. + * 3. A close whose durable steps FAIL still releases the lock (and still + * rethrows the failure). + * 4. A killed process (SIGKILL, no close at all) leaves the lock behind with + * NO record, and the next open says exactly that — crash, recovery ahead. + * 5. A host application with its own SIGTERM handler is never force-exited + * out from under its own shutdown by Brainy's handler. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs' +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const REPO_ROOT = process.cwd() +const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx') + +function makeTempDir(): string { + return mkdtempSync(join(tmpdir(), 'brainy-clean-close-')) +} + +/** + * Write a child script to disk and start it under tsx. A file (not `tsx -e`) + * because the eval form compiles to CommonJS, which has no top-level await. + * The script imports Brainy by ABSOLUTE path, so its own dependency + * resolution still happens from inside the repository. + */ +function startChild(dir: string, body: string): ReturnType { + const scriptPath = join(dir, 'child-process.mts') + writeFileSync(scriptPath, body) + // `detached` puts the child in its own process GROUP: tsx runs the script in + // a grandchild process, and only a group-wide signal reaches the process + // that actually holds the writer lock. + return spawn(TSX, [scriptPath], { + cwd: REPO_ROOT, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true + }) +} + +/** Capture every console.warn/error line emitted while `fn` runs. */ +async function captureConsole(fn: () => Promise): Promise<{ result: T; lines: string[] }> { + const lines: string[] = [] + const origWarn = console.warn + const origError = console.error + const sink = (...args: unknown[]) => { + lines.push(args.map((a) => String(a)).join(' ')) + } + console.warn = sink as typeof console.warn + console.error = sink as typeof console.error + try { + const result = await fn() + return { result, lines } + } finally { + console.warn = origWarn + console.error = origError + } +} + +/** + * Run a child process that opens `dir`, writes one row, prints `READY`, and + * then waits forever. Resolves with the child once READY is seen. + */ +function spawnHoldingChild(dir: string): Promise<{ + child: ReturnType + output: () => string +}> { + const script = ` + import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) + await brain.init() + await brain.add({ data: 'row from the child', type: 'concept' }) + await brain.flush() + console.log('READY') + setInterval(() => {}, 1000) + ` + const child = startChild(dir, script) + let out = '' + child.stdout.on('data', (d) => { out += String(d) }) + child.stderr.on('data', (d) => { out += String(d) }) + return new Promise((resolvePromise, rejectPromise) => { + const timer = setTimeout(() => rejectPromise(new Error(`child never became READY:\n${out}`)), 120_000) + child.stdout.on('data', () => { + if (out.includes('READY')) { + clearTimeout(timer) + resolvePromise({ child, output: () => out }) + } + }) + child.on('exit', (code) => { + clearTimeout(timer) + if (!out.includes('READY')) rejectPromise(new Error(`child exited ${code} before READY:\n${out}`)) + }) + }) +} + +describe('writer lock — the clean-close contract', () => { + let dir: string + let brain: Brainy | null = null + + beforeEach(() => { dir = makeTempDir() }) + + afterEach(async () => { + if (brain) { + try { await brain.close() } catch { /* may already be closed */ } + brain = null + } + try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } + }) + + const lockPath = () => join(dir, 'locks', '_writer.lock') + const recordPath = () => join(dir, 'locks', '_writer.close') + + it('a completed close leaves no lock, leaves a record, and the reopen is silent about staleness', async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + expect(existsSync(lockPath())).toBe(true) + + await brain.add({ data: 'seed entity', type: NounType.Concept }) + await brain.flush() + await brain.close() + brain = null + + // 1. The lock is gone and the release is RECORDED. + expect(existsSync(lockPath())).toBe(false) + expect(existsSync(recordPath())).toBe(true) + const record = JSON.parse(readFileSync(recordPath(), 'utf-8')) + expect(record.pid).toBe(process.pid) + expect(typeof record.closedAt).toBe('string') + expect(typeof record.startedAt).toBe('string') + + // 2. The reopen says nothing about a stale lock. + const { result: reopened, lines } = await captureConsole(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + return next + }) + brain = reopened + expect(lines.filter((l) => /stale writer lock|appears dead/i.test(l))).toEqual([]) + + // 3. The claim CONSUMED the record — it must not outlive its lock generation. + expect(existsSync(recordPath())).toBe(false) + expect(existsSync(lockPath())).toBe(true) + }, 120_000) + + it('releases the writer lock even when a durable close step fails — and still rethrows', async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + await brain.add({ data: 'seed entity', type: NounType.Concept }) + await brain.flush() + expect(existsSync(lockPath())).toBe(true) + + // Inject a failure into a durable close step (the counts flush). + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const boom = new Error('injected: counts flush failed during close') + storage.flushCounts = async () => { throw boom } + + await expect(brain.close()).rejects.toThrow(/injected: counts flush failed/) + brain = null + + // The lock is released regardless: a process on its way out holds nothing. + expect(existsSync(lockPath())).toBe(false) + + // And the next writer opens without a stale-lock verdict. + const { lines } = await captureConsole(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + await next.close() + }) + expect(lines.filter((l) => /appears dead/i.test(l))).toEqual([]) + }, 120_000) + + it('a SIGKILLed writer leaves the lock with no record, and the next open names the crash', async () => { + const { child } = await spawnHoldingChild(dir) + expect(existsSync(lockPath())).toBe(true) + expect(existsSync(recordPath())).toBe(false) + + // Group-wide: the lock holder is tsx's grandchild, not the spawned pid. + process.kill(-(child.pid as number), 'SIGKILL') + await new Promise((r) => child.on('exit', () => r())) + // The grandchild's death is asynchronous with the wrapper's exit event. + await new Promise((r) => setTimeout(r, 500)) + + // The lock survives the kill — a dead process releases nothing. + expect(existsSync(lockPath())).toBe(true) + expect(existsSync(recordPath())).toBe(false) + + const { lines } = await captureConsole(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + await next.close() + }) + const verdict = lines.filter((l) => /Overwriting stale writer lock/i.test(l)) + expect(verdict.length).toBe(1) + // The verdict must name the ABSENT record and the recovery it implies — + // not merely that a pid is gone. + expect(verdict[0]).toMatch(/NO\s+clean-close record/i) + expect(verdict[0]).toMatch(/crash recovery/i) + }, 180_000) + + it("does not force-exit a host application that owns its own SIGTERM handler", async () => { + const script = ` + import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) + await brain.init() + await brain.add({ data: 'row from the host app', type: 'concept' }) + await brain.flush() + // The host application's OWN graceful shutdown, registered after Brainy's. + process.on('SIGTERM', async () => { + await new Promise((r) => setTimeout(r, 1500)) + console.log('APP-CLOSE-DONE') + process.exit(0) + }) + console.log('READY') + setInterval(() => {}, 1000) + ` + const child = startChild(dir, script) + let out = '' + child.stdout.on('data', (d) => { out += String(d) }) + child.stderr.on('data', (d) => { out += String(d) }) + await new Promise((r, reject) => { + const timer = setTimeout(() => reject(new Error(`child never became READY:\n${out}`)), 120_000) + child.stdout.on('data', () => { if (out.includes('READY')) { clearTimeout(timer); r() } }) + child.on('exit', () => { clearTimeout(timer); if (!out.includes('READY')) reject(new Error(`child died:\n${out}`)) }) + }) + + process.kill(-(child.pid as number), 'SIGTERM') + const code = await new Promise((r) => child.on('exit', (c) => r(c))) + expect(code).toBe(0) + // The host's own shutdown ran to completion — Brainy's handler did not + // exit the process out from under it. + expect(out).toContain('APP-CLOSE-DONE') + }, 180_000) +}) From afe08a1ff990ed451caad2a673f7147e59fc3567 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:19:55 -0700 Subject: [PATCH 11/42] feat(open): the open narrates itself, on a channel production cannot clamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An operator watched a production service open a 16 GB store and print nothing for three minutes before its first line of work. Two defects, both fixed here. The narration was written to `prodLog.warn`, and every environment that looks like production clamps the logger to ERROR — so the phase breakdown that would have named the slow phase was composed and thrown away. `prodLog.narrate` is always visible, like `error`: it carries the two things an operator is entitled to hear from a database regardless of a cost setting — why it is slow and what it is doing about it. `silent: true` still silences it; that is a request, not a default. And nothing spoke DURING a phase, only after the whole open. init() now runs an unref'd heartbeat that every 5s names the phase currently running, its elapsed wall and what it is paying for, plus one line per phase as it ends for any phase over 2s. The generation-log fold's own progress and completion lines move to the same channel and now carry their wall — they were invisible in production, which is how an operator came to restart a converging fold three times. Pins: tests/integration/open-narration.test.ts — narrate() survives the clamp that silences warn(); a 6.5s storage-init produces a heartbeat naming the phase and a completion line naming its wall, with the logger clamped to ERROR. --- src/brainy.ts | 70 ++++++++++++-- src/db/generationStore.ts | 13 ++- src/utils/logger.ts | 20 ++++ tests/integration/open-narration.test.ts | 114 +++++++++++++++++++++++ 4 files changed, 202 insertions(+), 15 deletions(-) create mode 100644 tests/integration/open-narration.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 957920f5..81f3cc7e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1098,21 +1098,67 @@ export class Brainy implements BrainyInterface { configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging } - // OPEN-PATH NARRATION: lightweight phase timing across the five - // named stretches of init — storage init / generation-store open+fold / - // index init+gate / VFS bootstrap / embedding-warm-started. Each - // `markPhase()` call records elapsed ms SINCE THE PREVIOUS checkpoint, - // so the buckets always sum to the pre-integration/warmOnOpen total. - // Silent under 2s; one `prodLog.warn` line naming every phase's ms - // above it, so the operator's next restart storm names its own slow - // phase instead of re-deriving it from a stack of raw timestamps. + // OPEN-PATH NARRATION: phase timing across the five named stretches of + // init — storage init / generation-store open+fold / index init+gate / + // VFS bootstrap / embedding-warm-started. Each `markPhase()` call records + // elapsed ms SINCE THE PREVIOUS checkpoint, so the buckets always sum to + // the pre-integration/warmOnOpen total. + // + // THE LAW THIS ENFORCES: an open is never silent for more than + // OPEN_HEARTBEAT_MS. A production service opening a 16 GB store logged + // NOTHING for three minutes and then began work — the operator could not + // tell a slow open from a hung one, and restarted into the same wall. + // Two mechanisms, both on the always-visible narration channel (the old + // breakdown used `prodLog.warn`, which production clamps away — that is + // why the three minutes were silent): + // - a heartbeat that names the phase currently running and its elapsed + // wall, every OPEN_HEARTBEAT_MS, for as long as the open lasts; + // - one line per phase AS IT ENDS, naming its wall and its cause, for + // any phase over OPEN_PHASE_NARRATE_MS. + // The heartbeat is unref'd and cleared in the `finally` below, so it can + // neither hold the process open nor outlive a failed init. It cannot fire + // inside a phase that blocks the event loop synchronously; such a phase + // must narrate its own progress (the generation-log fold does). + const OPEN_HEARTBEAT_MS = 5_000 + const OPEN_PHASE_NARRATE_MS = 2_000 + /** Phase order + what each one is paying for, quoted in its narration. */ + const OPEN_PHASES: ReadonlyArray<{ name: string; cause: string }> = [ + { name: 'storage-init', cause: 'opening the store and loading its count ledger' }, + { + name: 'generation-store-open-fold', + cause: 'opening the generation store: crash-recovery replay/fold, derived-family registration, format handshake' + }, + { name: 'index-init-gate', cause: 'constructing the derived indexes and gating them for serving' }, + { name: 'vfs-bootstrap', cause: 'bootstrapping the virtual filesystem' }, + { name: 'embedding-warm-started', cause: 'starting the background embedding warm' } + ] const initStart = Date.now() let lastPhaseCheckpoint = initStart + let currentPhaseIndex = 0 const phaseTimingsMs: Record = {} + const openHeartbeat: ReturnType = setInterval(() => { + const phase = OPEN_PHASES[currentPhaseIndex] + if (!phase) return + prodLog.narrate( + `[Brainy] open: still in phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` + + `"${phase.name}" after ${Math.round((Date.now() - lastPhaseCheckpoint) / 1000)}s ` + + `(${Math.round((Date.now() - initStart) / 1000)}s into the open) — ${phase.cause}` + ) + }, OPEN_HEARTBEAT_MS) + if (typeof openHeartbeat.unref === 'function') openHeartbeat.unref() const markPhase = (name: string): void => { const now = Date.now() - phaseTimingsMs[name] = now - lastPhaseCheckpoint + const elapsed = now - lastPhaseCheckpoint + phaseTimingsMs[name] = elapsed lastPhaseCheckpoint = now + const finished = OPEN_PHASES[currentPhaseIndex] + if (elapsed >= OPEN_PHASE_NARRATE_MS && finished && finished.name === name) { + prodLog.narrate( + `[Brainy] open: phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` + + `"${name}" finished in ${elapsed}ms — ${finished.cause}` + ) + } + currentPhaseIndex++ } try { @@ -1777,7 +1823,7 @@ export class Brainy implements BrainyInterface { const phaseList = Object.entries(phaseTimingsMs) .map(([name, ms]) => `${name}=${ms}ms`) .join(', ') - prodLog.warn( + prodLog.narrate( `[Brainy] slow open: ${totalOpenMs}ms total (${phaseList}) — see the ` + `phase breakdown above to find which one to investigate first` ) @@ -1839,6 +1885,10 @@ export class Brainy implements BrainyInterface { // log — a plain string interpolation discards both stack and cause. const message = error instanceof Error ? error.message : String(error) throw new Error(`Failed to initialize Brainy: ${message}`, { cause: error }) + } finally { + // The open is over — succeeded or failed. Stop the heartbeat here so a + // failed init never leaves a timer narrating a phase nobody is running. + clearInterval(openHeartbeat) } } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 81bed338..d925c9e0 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -652,6 +652,7 @@ export class GenerationStore { : 'WHOLE-LOG fold' : 'above-manifest replay' let replayed = 0 + const foldStartedAt = Date.now() const replayFact = async (fact: CommitFact): Promise => { for (const op of fact.ops) { let image: { metadata: unknown | null; vector: unknown | null } @@ -697,9 +698,10 @@ export class GenerationStore { } replayed++ if (replayed % 1000 === 0) { - prodLog.warn( + prodLog.narrate( `[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` + - `(at generation ${fact.generation}); do not restart, the fold is finite` + `in ${Date.now() - foldStartedAt}ms (at generation ${fact.generation}); ` + + `do not restart, the fold is finite` ) } if (fact.generation > this.committed) { @@ -714,7 +716,7 @@ export class GenerationStore { } } if (uncleanOpen) { - prodLog.warn( + prodLog.narrate( `[GenerationStore] log-authority recovery: ${foldKind} beginning ` + `(unclean shutdown detected) — streaming replay, bounded memory, ` + `progress every 1000 facts. Do not restart the process; a restart ` + @@ -737,9 +739,10 @@ export class GenerationStore { } await this.storage.writeRawObject(MANIFEST_PATH, manifest) await this.storage.syncRawObjects([MANIFEST_PATH]) - prodLog.warn( + prodLog.narrate( `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + - `canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost` + `canonical in ${Date.now() - foldStartedAt}ms (${foldKind}; committed at ` + + `${this.committed}) — an acked write is never lost` ) } // A recovery fold re-applied (and the barrier below re-syncs) every diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 5154d4fd..0d6b6594 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -266,6 +266,26 @@ export const prodLog = { console.error(message, ...args) }, + /** + * THE NARRATION CHANNEL — always visible, exactly like `error`. + * + * `warn`/`info`/`log` below are clamped to ERROR in any environment that + * looks like production (see isProductionEnvironment), which is the right + * default for chatter and the wrong one for the two things an operator is + * entitled to hear from a database no matter what: WHY IT IS SLOW and WHAT + * IT IS DOING ABOUT IT. A production service opening a 16 GB store spent + * three minutes emitting nothing at all — the phase timings that would have + * named the slow phase were written to `warn` and thrown away by the log + * level. Progress and cost narration goes here; it is never a per-record + * line, always a phase, a wall, or a bounded-cadence heartbeat. + * + * `silent: true` still silences it — that is the consumer's explicit + * request, not a cost default. + */ + narrate: (message?: any, ...args: any[]) => { + console.warn(message, ...args) + }, + // These are suppressed in production unless BRAINY_LOG_LEVEL is set warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args), info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args), diff --git a/tests/integration/open-narration.test.ts b/tests/integration/open-narration.test.ts new file mode 100644 index 00000000..95aba9f1 --- /dev/null +++ b/tests/integration/open-narration.test.ts @@ -0,0 +1,114 @@ +/** + * @module tests/integration/open-narration + * @description THE OPEN IS NEVER SILENT. + * + * A production service opened a 16 GB store and logged nothing at all for + * three minutes before its first line of work. Two defects made that possible + * and both are pinned here: + * + * 1. The phase breakdown was written to `prodLog.warn`, which every + * environment that looks like production clamps away. The narration + * channel (`prodLog.narrate`) is always visible, like `error`. + * 2. Nothing spoke DURING a phase — only after the whole open finished, if + * at all. A heartbeat now names the phase currently running and its + * elapsed wall while the open is still happening. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' +import { prodLog, configureLogger, LogLevel } from '../../src/utils/logger.js' + +function makeTempDir(): string { + return mkdtempSync(join(tmpdir(), 'brainy-open-narration-')) +} + +/** Capture console.warn lines emitted while `fn` runs. */ +async function captureWarn(fn: () => Promise): Promise<{ result: T; lines: string[] }> { + const lines: string[] = [] + const orig = console.warn + console.warn = ((...args: unknown[]) => { + lines.push(args.map((a) => String(a)).join(' ')) + }) as typeof console.warn + try { + return { result: await fn(), lines } + } finally { + console.warn = orig + } +} + +describe('open narration', () => { + let dir: string + let brain: Brainy | null = null + + beforeEach(() => { dir = makeTempDir() }) + + afterEach(async () => { + if (brain) { + try { await brain.close() } catch { /* already closed */ } + brain = null + } + try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } + }) + + it('narrate() survives the production log clamp that silences warn()', async () => { + // Exactly what isProductionEnvironment() does to the logger: level ERROR. + configureLogger({ level: LogLevel.ERROR }) + try { + const { lines } = await captureWarn(async () => { + prodLog.warn('[Brainy] this line is chatter and may be clamped') + prodLog.narrate('[Brainy] this line is why the database is slow') + }) + expect(lines.some((l) => /why the database is slow/.test(l))).toBe(true) + expect(lines.some((l) => /chatter/.test(l))).toBe(false) + } finally { + configureLogger({ level: LogLevel.INFO }) + } + }) + + it('names a slow phase as it ends, and heartbeats while it is still running', async () => { + // Seed a store, then reopen it with a deliberately slow storage init so + // the first phase crosses both the heartbeat and the narrate thresholds. + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + await brain.add({ data: 'seed entity', type: NounType.Concept }) + await brain.flush() + await brain.close() + brain = null + + const realInit = FileSystemStorage.prototype.init + FileSystemStorage.prototype.init = async function slowInit(this: FileSystemStorage) { + await new Promise((r) => setTimeout(r, 6_500)) + return realInit.call(this) + } + // Clamped to ERROR for the whole open: the narration must survive it. + configureLogger({ level: LogLevel.ERROR }) + try { + const { result, lines } = await captureWarn(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + return next + }) + brain = result + + // The heartbeat spoke DURING the phase, naming the phase and its cause. + const heartbeats = lines.filter((l) => /open: still in phase 1\/5 "storage-init"/.test(l)) + expect(heartbeats.length).toBeGreaterThanOrEqual(1) + expect(heartbeats[0]).toMatch(/loading its count ledger/) + + // And the phase named its own wall as it ended. + const ended = lines.filter((l) => /open: phase 1\/5 "storage-init" finished in \d+ms/.test(l)) + expect(ended.length).toBe(1) + + // The whole-open breakdown is on the same always-visible channel. + expect(lines.some((l) => /slow open: \d+ms total \(.*storage-init=/.test(l))).toBe(true) + } finally { + FileSystemStorage.prototype.init = realInit + configureLogger({ level: LogLevel.INFO }) + } + }, 120_000) +}) From f4e2d34b4e897274cbc33205b6e9b2779ea63be9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:28:25 -0700 Subject: [PATCH 12/42] fix(storage): a suspect count ledger heals itself, and counts.json is written atomically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a real store: the ALL-visibility ledger read 14,231 nouns against 14,056 identity records and 72,729 verbs against 72,679 — exactly that store's 25 noun and 50 verb SCAR directories. Two copies of the same archive derived different numbers (14,231 and 14,081), because each had been persisted at a different moment under the old rule that counted one entity per id DIRECTORY. A downstream index heal subtracted against that denominator and reported remaining work that did not exist. The scan already applies the right predicate — one entity per IDENTITY RECORD (the metadata content leg), shared with pruneOrphanedEntities so the two agree by construction. What was missing is that a ledger persisted under the old rule was only FLAGGED suspect and then went on serving its wrong numbers for the life of the store, waiting for an operator to run repairIndex. - The ledger now derives itself honestly in the BACKGROUND after the open, narrating start and finish with the correction it made. Background because these scalars are denominators — no read is served from them — and because walks exactly like these are how a 24,898-id store spent minutes of a restart in silence. Observable via whenCountLedgerSettled(); nothing in the read path waits on it. - A derivation that raced a write refuses to stamp its number "exact": one retry on a quiet store, then the ledger stays SUSPECT and says so, naming repairIndex as the door that recounts under a barrier. - The one derivation that CANNOT leave the foreground says why it cannot: getNounCount()/getVerbCount() are served from it, and a background walk would make a populated store answer "0 entities" — a wrong answer, not a slow one. It narrates its start and its wall instead. - counts.json is written temp+rename. A truncating write left a window — measured at roughly 750ms after a flush or close — in which a concurrent reader saw the file EMPTY; an unparseable ledger sends the next open down the full-rescan path, so the cheapest file in the store was buying the most expensive recovery. - The writer lock's clean-close record is now consulted before the same-process branch too: a restart reported "Re-acquiring writer lock ... this is a bug" immediately after a clean close, sending an operator after a leak that did not exist. Pins: tests/integration/count-ledger-identity-record.test.ts (background correction with scar and ghost fixtures, two copies of one archive agreeing, counts.json never observed unparseable across 40 persists); tests/integration/ledger-derivation-identity.test.ts updated to the new law — the OPEN still never walks (proved by slowing the walk 1.2s and timing the open), and the ledger heals behind it. --- src/storage/adapters/fileSystemStorage.ts | 253 +++++++++++++++--- .../count-ledger-identity-record.test.ts | 251 +++++++++++++++++ .../ledger-derivation-identity.test.ts | 87 ++++-- 3 files changed, 519 insertions(+), 72 deletions(-) create mode 100644 tests/integration/count-ledger-identity-record.test.ts diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index a01c2015..784ea5d8 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -120,6 +120,13 @@ export class FileSystemStorage extends BaseStorage { */ private writerHeartbeatInFlight?: Promise + /** + * The in-flight background count-ledger derivation, if one was needed at + * open. See {@link scheduleCountLedgerDerivation} — awaited only by + * {@link whenCountLedgerSettled}, never by a read. + */ + private countLedgerDerivation?: Promise + // Flush-request RPC state. The writer polls `locks/_flush_requests/` for // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after // flushing. Inspectors call `requestFlushOverFilesystem` to drop a request @@ -1889,18 +1896,41 @@ export class FileSystemStorage extends BaseStorage { } } + // THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see + // WriterCloseRecord). A lock file whose release was RECORDED is + // bookkeeping left by an orderly shutdown, not evidence of anything — + // and that is true whether the previous holder was another process or + // an earlier instance in THIS one. A production restart reported + // "Re-acquiring writer lock ... this is a bug" immediately after a clean + // close, sending an operator hunting for a leak that did not exist. + const closeRecord = existing ? await this.readWriterCloseRecord() : null + const releasedCleanly = + existing !== null && + closeRecord !== null && + this.closeRecordVouchesFor(closeRecord, existing) + if (existing) { // Same-process re-open: a second Brainy instance in this Node process // (e.g. test "simulate server restart" patterns, or a consumer that // explicitly re-instantiates without closing first). This isn't the // dangerous cross-process case the lock exists to prevent — the two // instances share a memory space and can't silently diverge from each - // other beyond what their callers already see. Warn and take over. + // other beyond what their callers already see. Warn and take over — + // unless the record proves the previous instance already let go, in + // which case there is nothing to warn about. if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { - console.warn( - `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + - `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` - ) + if (releasedCleanly) { + console.warn( + `[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` + + `instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` + + `${closeRecord!.closedAt} but could not remove the file. Nothing to recover.` + ) + } else { + console.warn( + `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` + ) + } const info: WriterLockInfo = { pid: myPid, hostname, @@ -1915,17 +1945,11 @@ export class FileSystemStorage extends BaseStorage { return info } - // THE CLEAN-CLOSE RECORD IS CONSULTED FIRST (see WriterCloseRecord). - // A lock file whose release was RECORDED is bookkeeping left behind by - // an orderly shutdown, not evidence of a crash — take it over calmly - // and say so. Only when no record vouches for this lock do we fall - // back to inferring liveness from the pid, and then we say THAT - // honestly too: an unrecorded lock means the writer did not complete - // its close, so the store was not closed cleanly and this open pays - // recovery. - const closeRecord = await this.readWriterCloseRecord() - const releasedCleanly = - closeRecord !== null && this.closeRecordVouchesFor(closeRecord, existing) + // A cleanly-released lock is stale by RECORD, not by inference. Only + // when no record vouches for this lock do we fall back to pid + // liveness, and then we say THAT honestly too: an unrecorded lock + // means the writer did not complete its close, so the store was not + // closed cleanly and this open pays recovery. const stale = releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing))) if (!options?.force && !stale) { @@ -2224,6 +2248,9 @@ export class FileSystemStorage extends BaseStorage { try { await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2)) } catch (err) { + // ENOENT = the lock directory is gone, i.e. the whole store was removed + // under us. There is no next open to inform. + if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return console.warn( `[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` + `the next open will fall back to pid liveness and may report this orderly ` + @@ -2760,25 +2787,29 @@ export class FileSystemStorage extends BaseStorage { this.allCountsDerivedBy = undefined this.allCountsSuspect = true needsPersist = true - prodLog.warn( + prodLog.narrate( '[FileSystemStorage] canonical count ledger was derived under the legacy ' + - 'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' + - 'exact denominators' + 'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' + + 'container inflates it. Marked suspect, and an honest recount is scheduled to ' + + 'run in the background after this open; until it lands, do not subtract ' + + 'against these ALL scalars.' ) + // A suspect ledger used to stay wrong for the life of the store, + // waiting for an operator to run repairIndex. A downstream index + // heal took its "remaining" figure from these inflated + // denominators and reported work that did not exist. The ledger + // now HEALS ITSELF — in the background, because a denominator is + // a derived scalar and no read is ever served from it. + this.scheduleCountLedgerDerivation('legacy container-rule ledger') } } else { - const nouns = await this.scanCanonicalEntities('nouns') - const verbs = await this.scanCanonicalEntities('verbs') - this.totalNounCountAll = nouns.count - this.totalVerbCountAll = verbs.count - this.allCountsSuspect = false - this.allCountsDerivedBy = 'identity-record' - console.warn( - `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` + - `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + - `every tier) and persisted; no further scan.` - ) - needsPersist = true + // No ALL scalars at all. There is nothing to serve in the meantime — + // a zero would read as an empty store — so the scalars stay unknown + // and SUSPECT until the background derivation lands. The open does + // not wait for it: an id-tree walk is O(ids) and this file has been + // the whole reason a 24k-id store opened in silence. + this.allCountsSuspect = true + this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger') } // The vectored-noun scalar (shipped after the ALL scalars above — a @@ -2791,14 +2822,12 @@ export class FileSystemStorage extends BaseStorage { if (typeof counts.totalVectoredNounCount === 'number') { this.totalVectoredNounCount = counts.totalVectoredNounCount } else { - const vectored = await this.scanVectoredNounCount() - this.totalVectoredNounCount = vectored - console.warn( - `[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` + - `derived once by reading every noun's vectors.json (${vectored} vectored) and ` + - `persisted; no further scan.` - ) - needsPersist = true + // O(nouns) CONTENT reads — the most expensive derivation of the + // three, and the one most likely to have been the silent minutes at + // the front of a large store's open. Background, suspect until it + // lands, same as the ALL scalars. + this.allCountsSuspect = true + this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger') } if (needsPersist) { await this.persistCounts() @@ -2827,6 +2856,22 @@ export class FileSystemStorage extends BaseStorage { * Initialize counts by scanning disk (only done once) */ private async initializeCountsFromDisk(): Promise { + const startedAt = Date.now() + // THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating: + // it derives `totalNounCount` / `totalVerbCount`, the scalars + // `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would + // make a populated store answer "0 entities" until the walk landed — a + // wrong answer, not a slow one, and the serving law grades a failure by + // whether an answer could be wrong. The ALL-visibility denominators, which + // no read is served from, DO run in the background (see + // scheduleCountLedgerDerivation). What this walk owes the operator instead + // is narration: it announces itself, and reports its wall. + prodLog.narrate( + `[FileSystemStorage] no usable counts.json — deriving the entity counters from ` + + `the canonical id tree now. This is O(ids) listings plus one vectors.json read ` + + `per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` + + `served from it. It runs once; the result is persisted.` + ) try { // Count the CANONICAL 8.0 layout (`entities////…`) — // the tree saveNoun/getNouns actually read and write. The previous scan @@ -2874,6 +2919,11 @@ export class FileSystemStorage extends BaseStorage { } await this.persistCounts() + prodLog.narrate( + `[FileSystemStorage] counter derivation from the canonical id tree finished in ` + + `${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` + + `${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.` + ) } catch (error) { console.error('Error initializing counts from disk:', error) } @@ -2895,6 +2945,118 @@ export class FileSystemStorage extends BaseStorage { * directories (absolute paths) — nouns feed the type-distribution estimate * above. An absent tree (fresh store) counts zero. */ + /** + * @description Derive the ALL-visibility count ledger honestly — one entity + * per IDENTITY RECORD, never per id directory — IN THE BACKGROUND, once, + * and persist the result stamped `identity-record`. + * + * Why background: these scalars are DENOMINATORS. No read is served from + * them, so deriving them cannot be allowed to hold an open hostage — a + * store with 24,898 ids spent minutes of a production restart inside walks + * exactly like these, in silence, before serving anything. Why at all: a + * ledger derived under the old container rule stayed wrong for the life of + * the store, and a downstream index heal subtracted against it and reported + * remaining work that did not exist (measured on a real store: 14,231 + * derived against 14,056 identity records — precisely the store's 25 noun + * scar directories; verbs 72,729 against 72,679, its 50 verb scars). + * + * Idempotent: a second call while one is in flight joins the first. + * @param reason - What made the ledger untrustworthy, quoted in narration. + * @returns Nothing; observe completion with {@link whenCountLedgerSettled}. + */ + private scheduleCountLedgerDerivation(reason: string): void { + if (this.countLedgerDerivation) return + this.countLedgerDerivation = (async () => { + const startedAt = Date.now() + prodLog.narrate( + `[FileSystemStorage] count-ledger derivation started in the background ` + + `(${reason}) — counting identity records, not id directories; the open does ` + + `not wait for it and no read is served from these scalars.` + ) + try { + const beforeNouns = this.totalNounCountAll + const beforeVerbs = this.totalVerbCountAll + const beforeVectored = this.totalVectoredNounCount + // A walk that RACED A WRITE cannot prove its number: a row that landed + // mid-walk may or may not have been in the shard the walk had already + // passed. Rather than persist a figure that might be off by one and + // stamp it "exact", the walk is repeated once on a quiet store, and if + // the store is never quiet the ledger stays SUSPECT and says so. One + // retry, never a spin. + let attempt = 0 + let derived: { nouns: number; verbs: number; vectored: number } | null = null + while (attempt < 2 && derived === null) { + attempt++ + const activityBefore = this.ledgerActivityStamp() + const nouns = await this.scanCanonicalEntities('nouns') + const verbs = await this.scanCanonicalEntities('verbs') + const vectored = await this.scanVectoredNounCount() + if (this.ledgerActivityStamp() === activityBefore) { + derived = { nouns: nouns.count, verbs: verbs.count, vectored } + } + } + if (derived === null) { + this.allCountsSuspect = true + prodLog.narrate( + `[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` + + `after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` + + `every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` + + `against; brain.repairIndex() derives them under a recount barrier.` + ) + return + } + this.totalNounCountAll = derived.nouns + this.totalVerbCountAll = derived.verbs + this.totalVectoredNounCount = derived.vectored + this.allCountsDerivedBy = 'identity-record' + this.allCountsSuspect = false + await this.persistCounts() + prodLog.narrate( + `[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` + + `${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` + + (beforeNouns !== derived.nouns || + beforeVerbs !== derived.verbs || + beforeVectored !== derived.vectored + ? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` + + `difference is ghost and scar containers the old rule counted as entities)` + : ' (unchanged)') + + ` — persisted, stamped identity-record, no longer suspect.` + ) + } catch (error) { + // The ledger stays suspect and the next open retries. Loud: a + // denominator nobody can derive is a fact an operator must have. + this.allCountsSuspect = true + prodLog.error( + `[FileSystemStorage] count-ledger derivation FAILED after ` + + `${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` + + `and must not be subtracted against; the next open retries:`, + error + ) + } + })() + } + + /** + * @description A cheap witness that the ledger changed while a walk was + * running. Every landed write moves one of these live counters, so an + * unchanged stamp across a walk means no write landed during it. + * @returns A value that differs whenever the live ALL scalars have moved. + */ + private ledgerActivityStamp(): string { + return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}` + } + + /** + * @description Resolve once any background count-ledger derivation has + * settled (succeeded or failed). Resolves immediately when none was needed. + * Exists so tests and operators can observe the ledger's honest value rather + * than race it; nothing in the read path waits on this. + * @returns A promise that settles with the derivation. + */ + public async whenCountLedgerSettled(): Promise { + await this.countLedgerDerivation + } + private async scanCanonicalEntities( kind: 'nouns' | 'verbs' ): Promise<{ count: number; sampleDirs: string[] }> { @@ -3053,10 +3215,15 @@ export class FileSystemStorage extends BaseStorage { lastUpdated: new Date().toISOString() } - await fs.promises.writeFile( - this.countsFilePath, - JSON.stringify(counts, null, 2) - ) + // ATOMIC (temp + rename), never a plain writeFile. A direct write + // truncates the file first, so every persist opened a window — measured + // at roughly 750ms after a flush or close on a real store — in which a + // concurrent reader saw counts.json EMPTY. An empty file is unparseable, + // and an unparseable ledger sends the next open down the full-rescan + // path: the cheapest file in the store was costing the most expensive + // recovery. The rename is atomic, so a reader sees the old ledger or the + // new one, never neither. + await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2)) } catch (error) { console.error('Error persisting counts:', error) } diff --git a/tests/integration/count-ledger-identity-record.test.ts b/tests/integration/count-ledger-identity-record.test.ts new file mode 100644 index 00000000..1066213a --- /dev/null +++ b/tests/integration/count-ledger-identity-record.test.ts @@ -0,0 +1,251 @@ +/** + * @module tests/integration/count-ledger-identity-record + * @description THE COUNT LEDGER COUNTS RECORDS, NOT DIRECTORIES — and heals + * itself when it was derived the other way. + * + * Measured on a real store: the ALL-visibility ledger read 14,231 nouns + * against 14,056 identity records, and 72,729 verbs against 72,679 — exactly + * that store's 25 noun and 50 verb SCAR directories (empty `/` containers + * left by a pre-8.3.1 partial delete). Two copies of the SAME archive derived + * different numbers, because each had been persisted at a different moment + * under the old container rule. A downstream index heal subtracted against + * those denominators and reported remaining work that did not exist. + * + * The membership predicate is the IDENTITY RECORD (the metadata content leg). + * The scan already applies it; what is pinned here is that a ledger persisted + * under the OLD rule does not go on lying — it is corrected in the background, + * without blocking the open, and two copies of one archive agree. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { + mkdtempSync, + mkdirSync, + rmSync, + writeFileSync, + readFileSync, + cpSync, + existsSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { FileSystemStorage as FileSystemStorageClass } from '../../src/storage/adapters/fileSystemStorage.js' +import type { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' + +const NOUN_COUNT = 6 +const NOUN_SCARS = 3 +const VERB_SCARS = 2 +/** A REAL two-hex shard — the scan skips any directory that is not one. */ +const SCAR_SHARD = 'ab' + +function makeTempDir(): string { + return mkdtempSync(join(tmpdir(), 'brainy-count-ledger-')) +} + +/** The FileSystemStorage behind a brain. */ +function storageOf(brain: Brainy): FileSystemStorage { + return (brain as unknown as { storage: FileSystemStorage }).storage +} + +/** + * Add `count` empty `/` container directories under + * `entities///` — scars, exactly as a partial delete leaves them. + */ +function addScarContainers(dir: string, kind: 'nouns' | 'verbs', count: number): void { + for (let i = 0; i < count; i++) { + const id = `${SCAR_SHARD}5ca4000-0000-0000-0000-00000000000${i}` + mkdirSync(join(dir, 'entities', kind, SCAR_SHARD, id), { recursive: true }) + } +} + +/** Add one GHOST container: a `vectors.json` leg with no identity record. */ +function addGhostContainer(dir: string): void { + const id = `${SCAR_SHARD}9405700-0000-0000-0000-000000000000` + const idDir = join(dir, 'entities', 'nouns', SCAR_SHARD, id) + mkdirSync(idDir, { recursive: true }) + writeFileSync(join(idDir, 'vectors.json'), JSON.stringify({ id, vector: [0.1, 0.2] })) +} + +/** + * Rewrite counts.json into the LEGACY shape: ALL scalars inflated by the + * containers, and no `allCountsDerivedBy` stamp — exactly what a store carried + * when it was last written by a build that counted directories. + */ +function writeLegacyCountsLedger(dir: string, inflateNouns: number, inflateVerbs: number): void { + const file = join(dir, '_system', 'counts.json') + const counts = JSON.parse(readFileSync(file, 'utf-8')) + counts.totalNounCountAll = (counts.totalNounCountAll ?? 0) + inflateNouns + counts.totalVerbCountAll = (counts.totalVerbCountAll ?? 0) + inflateVerbs + delete counts.allCountsDerivedBy + delete counts.allCountsSuspect + writeFileSync(file, JSON.stringify(counts, null, 2)) +} + +/** + * Seed a store and return the HONEST ledger it holds when freshly written — + * the baseline the correction must return to. Read from the engine rather than + * hardcoded: an open creates its own rows (the VFS root), and a pin that + * asserts a literal would be pinning that incidental fact instead of the rule. + */ +async function seedStore(dir: string): Promise<{ nouns: number; verbs: number }> { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + const ids: string[] = [] + for (let i = 0; i < NOUN_COUNT; i++) { + ids.push(await brain.add({ data: `entity number ${i}`, type: NounType.Concept })) + } + await brain.relate({ from: ids[0], to: ids[1], type: 'relatedTo' } as never) + await brain.relate({ from: ids[1], to: ids[2], type: 'relatedTo' } as never) + await brain.flush() + const ledger = await storageOf(brain).getCanonicalCounts() + const baseline = { nouns: ledger.nouns.all, verbs: ledger.verbs.all } + await brain.close() + return baseline +} + +/** + * Make the ledger walk take `ms` so a test can observe the open completing + * WITHOUT it. Patches the prototype before any brain is constructed; returns + * the restore function. + */ +function slowTheLedgerWalk(ms: number): () => void { + const proto = ( + FileSystemStorageClass as unknown as { + prototype: Record Promise> + } + ).prototype + const real = proto.scanCanonicalEntities + proto.scanCanonicalEntities = async function slow(this: unknown, ...args: unknown[]) { + await new Promise((r) => setTimeout(r, ms)) + return real.apply(this, args) + } + return () => { proto.scanCanonicalEntities = real } +} + +describe('the canonical count ledger', () => { + const dirs: string[] = [] + + afterEach(() => { + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + function trackDir(): string { + const dir = makeTempDir() + dirs.push(dir) + return dir + } + + it('corrects a legacy container-rule ledger in the background, counting identity records', async () => { + const dir = trackDir() + const baseline = await seedStore(dir) + + // Scars and a ghost: containers with no identity record. + addScarContainers(dir, 'nouns', NOUN_SCARS) + addScarContainers(dir, 'verbs', VERB_SCARS) + addGhostContainer(dir) + // The ledger as the old rule left it: every container counted. + writeLegacyCountsLedger(dir, NOUN_SCARS + 1, VERB_SCARS) + + const restore = slowTheLedgerWalk(1_500) + let brain: Brainy + try { + const openStarted = Date.now() + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + const openMs = Date.now() - openStarted + const storage = storageOf(brain) + + // THE OPEN DID NOT WAIT. Two walks of 1.5s each would have added 3s. + expect(openMs).toBeLessThan(2_500) + // And while it runs, the scalars say so instead of being subtracted against. + const atOpen = await storage.getCanonicalCounts() + expect(atOpen.suspect).toBe(true) + expect(atOpen.nouns.all).toBe(baseline.nouns + NOUN_SCARS + 1) + + await storage.whenCountLedgerSettled() + } finally { + restore() + } + const storage = storageOf(brain!) + + const healed = await storage.getCanonicalCounts() + expect(healed.nouns.all).toBe(baseline.nouns) + expect(healed.verbs.all).toBe(baseline.verbs) + expect(healed.suspect).toBe(false) + + // And it is PERSISTED with the honest stamp — the correction survives a + // reopen instead of being re-derived (or re-lost) every time. + await brain!.close() + const persisted = JSON.parse(readFileSync(join(dir, '_system', 'counts.json'), 'utf-8')) + expect(persisted.totalNounCountAll).toBe(baseline.nouns) + expect(persisted.totalVerbCountAll).toBe(baseline.verbs) + expect(persisted.allCountsDerivedBy).toBe('identity-record') + + const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await reopened.init() + const afterReopen = await storageOf(reopened).getCanonicalCounts() + expect(afterReopen.nouns.all).toBe(baseline.nouns) + expect(afterReopen.suspect).toBe(false) + await reopened.close() + }, 180_000) + + it('derives the same number from two copies of one archive', async () => { + const source = trackDir() + const baseline = await seedStore(source) + addScarContainers(source, 'nouns', NOUN_SCARS) + addGhostContainer(source) + + // Two copies of the SAME bytes, each carrying a DIFFERENT legacy ledger — + // the situation that made one archive report 14,231 and its twin 14,081. + const copyA = trackDir() + const copyB = trackDir() + cpSync(source, copyA, { recursive: true }) + cpSync(source, copyB, { recursive: true }) + writeLegacyCountsLedger(copyA, NOUN_SCARS + 1, 0) + writeLegacyCountsLedger(copyB, 1, 0) + + const derived: number[] = [] + for (const dir of [copyA, copyB]) { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + const storage = storageOf(brain) + await storage.whenCountLedgerSettled() + derived.push((await storage.getCanonicalCounts()).nouns.all) + await brain.close() + } + expect(derived[0]).toBe(derived[1]) + expect(derived[0]).toBe(baseline.nouns) + }, 180_000) + + it('writes counts.json atomically — no reader ever sees it empty', async () => { + const dir = trackDir() + await seedStore(dir) + const file = join(dir, '_system', 'counts.json') + expect(existsSync(file)).toBe(true) + + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + const storage = storageOf(brain) + + // Watch the ledger across many persists. A truncating write leaves a + // window in which the file parses as nothing; a temp+rename never does. + let sawUnparseable = 0 + const watcher = setInterval(() => { + try { + JSON.parse(readFileSync(file, 'utf-8')) + } catch { + sawUnparseable++ + } + }, 1) + for (let i = 0; i < 40; i++) { + await (storage as unknown as { persistCounts: () => Promise }).persistCounts() + } + clearInterval(watcher) + await brain.close() + expect(sawUnparseable).toBe(0) + }, 180_000) +}) diff --git a/tests/integration/ledger-derivation-identity.test.ts b/tests/integration/ledger-derivation-identity.test.ts index cb19af4a..7d09e893 100644 --- a/tests/integration/ledger-derivation-identity.test.ts +++ b/tests/integration/ledger-derivation-identity.test.ts @@ -11,14 +11,22 @@ * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per * metadata content leg (`metadata.json` or `.json.gz`), the same test * `pruneOrphanedEntities()` uses, so the two agree by construction. - * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that - * carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'` - * stamp predates this fix; loading it marks `suspect = true` from a - * single field read alone, never a directory walk, and warns exactly - * once naming the cause. - * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned - * containers, recounts from the canonical metadata.json walk, and - * re-stamps — suspect clears and the ALL scalar is exact again. + * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AND THE OPEN NEVER WALKS — a + * counts.json that carries the ALL scalars but no + * `allCountsDerivedBy: 'identity-record'` stamp predates this fix; + * loading it marks `suspect = true` from a single field read alone and + * warns exactly once naming the cause. The open itself never pays a + * directory walk. + * (2b) AND IT HEALS ITSELF. The ledger used to stay wrong for the life of the + * store, waiting for an operator to run `repairIndex()` — and a + * downstream index heal subtracted against the inflated denominator and + * reported work that did not exist. An honest derivation now runs in the + * BACKGROUND after the open (never blocking it, observable via + * `whenCountLedgerSettled()`), and refuses to stamp a number it derived + * while writes were landing. + * (3) THE SANCTIONED RECOUNT ALSO CLEARS IT — `repairIndex()` prunes the + * orphaned containers, recounts from the canonical metadata.json walk, + * and re-stamps — the ALL scalar is exact and the containers are gone. * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store * with no counts.json stamps as it writes, so a brand-new store never * carries the legacy signature. @@ -115,29 +123,43 @@ describe('ledger derivation identity — the ALL scalar is the identity-record p delete raw.allCountsDerivedBy fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) - const warnSpy = vi.spyOn(prodLog, 'warn') - // The two derivation walks live on FileSystemStorage's prototype — - // spying here (rather than on fs.promises.readdir globally) isolates - // THIS code path's behavior from unrelated walks elsewhere in the open - // sequence (a separate, pre-existing engine's own O(store) cost — not - // this fix's concern, and not something this pin should be sensitive - // to). Neither derivation method may run: the stamp check is a field - // read on the already-parsed counts.json, nothing more. - const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities') - const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount') + const narrateSpy = vi.spyOn(prodLog, 'narrate') + // The derivation walks live on FileSystemStorage's prototype. Slow them + // deliberately: the OPEN must not wait for them, and on a two-row store a + // real walk finishes too fast to tell "not awaited" from "instant". + const proto = FileSystemStorage.prototype as any + const realScanEntities = proto.scanCanonicalEntities + let scanEntitiesCalls = 0 + proto.scanCanonicalEntities = async function slow(this: any, ...args: any[]) { + scanEntitiesCalls++ + await new Promise((r) => setTimeout(r, 1_200)) + return realScanEntities.apply(this, args) + } + try { + const openStarted = Date.now() + brain = await open() + const openMs = Date.now() - openStarted - brain = await open() + // THE OPEN DID NOT WALK: two slowed walks would have added 2.4s to it. + expect(openMs).toBeLessThan(2_000) - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(true) + // The stamp check itself is an O(1) field read, and it names the cause. + const atOpen = await brain.storage.getCanonicalCounts() + expect(atOpen.suspect).toBe(true) + const stampWarnings = narrateSpy.mock.calls.filter( + ([msg]: any[]) => String(msg).includes('legacy') && String(msg).includes('container rule') + ) + expect(stampWarnings.length).toBe(1) // exactly one, loud - const stampWarnings = warnSpy.mock.calls.filter( - ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule') - ) - expect(stampWarnings.length).toBe(1) // exactly one, loud - - expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk - expect(scanVectoredSpy).not.toHaveBeenCalled() + // ...and the honest derivation is already running behind the open. + await brain.storage.whenCountLedgerSettled() + expect(scanEntitiesCalls).toBeGreaterThan(0) + const healed = await brain.storage.getCanonicalCounts() + expect(healed.suspect).toBe(false) + expect(healed.nouns.all).toBe(raw.totalNounCountAll) + } finally { + proto.scanCanonicalEntities = realScanEntities + } await brain.close() }) @@ -163,7 +185,14 @@ describe('ledger derivation identity — the ALL scalar is the identity-record p fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) brain = await open() - expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load + // Named suspect at load, then healed in the background WITHOUT the + // operator asking — the inflated container count is corrected to the + // identity-record population, though the orphaned containers themselves + // are still on disk (only repairIndex() removes those). + await brain.storage.whenCountLedgerSettled() + let healed = await brain.storage.getCanonicalCounts() + expect(healed.suspect).toBe(false) + expect(healed.nouns.all).toBe(realTotal) await brain.repairIndex() From 3fffd9c6e66f6e67b1eae203f470da3312427d00 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:31:42 -0700 Subject: [PATCH 13/42] feat(repair): repairIndex narrates every phase and its receipt carries the walls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a production store (14,647 nouns / 73,070 verbs) a repairIndex() ran for more than thirty minutes at roughly a full core with ZERO log lines between its start and its end, while the read doors kept serving. The operator could tell it was alive only from `top`, and could not tell which of its single-threaded walks it was inside. Same law as the open, applied to the repair: - every phase announces itself BEFORE it works, naming what it is about to walk (each canonical walk, the VFS containment reconciliation, each provider's invariant pass); - an unref'd heartbeat names the phase still running every 5s, for as long as it runs; - every phase reports its own wall, and that wall is carried in the TYPED receipt as RepairFamilyReport.durationMs — a receipt that cannot say where the time went is not a receipt; - the whole repair's narration moves to the always-visible channel, so a production log level cannot silence it. The phases move into runRepairIndexPhases() so the heartbeat can live in a finally around them; the public door and its report shape are unchanged apart from the added durationMs. Pins: tests/integration/repair-narration.test.ts — every checked family has a start line, a finish line with its wall, and a numeric durationMs in the receipt; a phase slowed to 6.5s produces a heartbeat naming it, with the logger clamped to ERROR. --- src/brainy.ts | 128 ++++++++++++++++++--- src/types/brainy.types.ts | 7 ++ tests/integration/repair-narration.test.ts | 119 +++++++++++++++++++ 3 files changed, 240 insertions(+), 14 deletions(-) create mode 100644 tests/integration/repair-narration.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 81f3cc7e..24407018 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -18199,17 +18199,89 @@ export class Brainy implements BrainyInterface { * invariant-driven pass (it was already rebuilt unconditionally — a second, * report-driven pass over the same family would be redundant at best). * + * NARRATION IS PART OF THE CONTRACT. A repair on a production store ran for + * more than thirty minutes at a full core with NOT ONE log line between its + * start and its end while the doors kept serving; the operator could tell it + * was alive only from `top`. Every phase now announces itself before it + * works, a heartbeat names the phase still running every five seconds, and + * each phase reports its own wall — carried in the receipt as + * `durationMs` per family, so nobody has to infer progress from CPU. + * * @param options.rebuild - Family name(s) to unconditionally rebuild, or `'all'` for all three (`'metadata' | 'graph' | 'vector'`). - * @returns The full per-family receipt (see {@link RepairReport}); also narrated via `prodLog.warn`. + * @returns The full per-family receipt (see {@link RepairReport}); also narrated as it goes. */ async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { await this.ensureInitialized() const startedAt = Date.now() const families: RepairFamilyReport[] = [] - const record = (family: string, entry: Omit): void => { - families.push({ family, ...entry }) + + // THE REPAIR HEARTBEAT — the same law the open obeys: no stretch of work + // may be silent for more than REPAIR_HEARTBEAT_MS. Unref'd (it never holds + // a process open) and cleared in the `finally` below. + const REPAIR_HEARTBEAT_MS = 5_000 + let currentPhase = 'starting' + let currentPhaseCause = 'preparing the repair' + let phaseStartedAt = Date.now() + const heartbeat = setInterval(() => { + prodLog.narrate( + `[Brainy] repairIndex: still in "${currentPhase}" after ` + + `${Math.round((Date.now() - phaseStartedAt) / 1000)}s ` + + `(${Math.round((Date.now() - startedAt) / 1000)}s into the repair) — ${currentPhaseCause}` + ) + }, REPAIR_HEARTBEAT_MS) + if (typeof heartbeat.unref === 'function') heartbeat.unref() + + /** Announce a phase before it does any work, and start its clock. */ + const beginPhase = (name: string, cause: string): void => { + currentPhase = name + currentPhaseCause = cause + phaseStartedAt = Date.now() + prodLog.narrate(`[Brainy] repairIndex: "${name}" started — ${cause}`) } + /** + * Close the current phase: stamp its wall into the receipt row and say + * what it did. Every family row carries its own `durationMs`. + */ + const record = (family: string, entry: Omit): void => { + const durationMs = Date.now() - phaseStartedAt + families.push({ family, ...entry, durationMs }) + prodLog.narrate( + `[Brainy] repairIndex: "${family}" finished in ${durationMs}ms — ` + + (entry.checked + ? `${entry.healed} heal(s)${entry.rebuilt ? ', rebuilt' : ''}` + + (entry.detail ? ` (${entry.detail})` : '') + : `skipped (${entry.skipped ?? entry.reason ?? 'no reason given'})`) + ) + phaseStartedAt = Date.now() + } + + try { + return await this.runRepairIndexPhases(options, families, record, beginPhase, startedAt) + } finally { + clearInterval(heartbeat) + } + } + + /** + * @description The phases of {@link repairIndex}, separated so its heartbeat + * can live in a `finally` around them. Not a public door — see `repairIndex` + * for the contract. + * @param options - As `repairIndex`. + * @param families - The receipt rows being accumulated. + * @param record - Closes a phase: stamps its wall and narrates its outcome. + * @param beginPhase - Announces a phase before it works. + * @param startedAt - When the repair began, for the closing line. + * @returns The full receipt. + */ + private async runRepairIndexPhases( + options: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' } | undefined, + families: RepairFamilyReport[], + record: (family: string, entry: Omit) => void, + beginPhase: (name: string, cause: string) => void, + startedAt: number + ): Promise { + // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete // defect: a delete that removed the metadata (content) leg but left the // vector leg + the entity directory (a "ghost"), or left an empty directory @@ -18223,6 +18295,10 @@ export class Brainy implements BrainyInterface { rebuildSubtypeCounts?: () => Promise } if (typeof pruner.pruneOrphanedEntities === 'function') { + beginPhase( + 'orphaned-containers', + 'walking every canonical id directory for ghost/scar containers left by a partial delete' + ) const orphans = await pruner.pruneOrphanedEntities() const pruned = orphans.nouns.length + orphans.verbs.length record('orphaned-containers', { @@ -18233,7 +18309,7 @@ export class Brainy implements BrainyInterface { : {}) }) if (pruned > 0) { - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` + `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` + `partial delete.` @@ -18249,6 +18325,10 @@ export class Brainy implements BrainyInterface { // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup // (scalar totals + per-type maps + type-statistics arrays) from one // canonical walk and persists them. + beginPhase( + 'count-rollups', + 'ONE canonical walk recomputing every counter rollup — scalar totals, per-type maps, type statistics' + ) await pruner.rebuildTypeCounts?.() await pruner.rebuildSubtypeCounts?.() record('count-rollups', { @@ -18271,6 +18351,10 @@ export class Brainy implements BrainyInterface { // concurrent writers. Canonical metadata.path is the truth; only VFS // containment edges are touched. Loud per repair. if (this._vfsInitialized && this._vfs) { + beginPhase( + 'vfs-containment', + 'reconciling VFS containment edges against canonical metadata.path' + ) const containment = await this._vfs.repairContainment() record('vfs-containment', { checked: true, @@ -18280,7 +18364,7 @@ export class Brainy implements BrainyInterface { : {}) }) if (containment.removed + containment.restored > 0) { - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` + `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).` ) @@ -18291,17 +18375,25 @@ export class Brainy implements BrainyInterface { record('vfs-containment', { checked: false, healed: 0, skipped: 'VFS not initialized' }) } + beginPhase( + 'metadata-corruption', + 'detect-and-repair pass over the metadata index' + ) await this.metadataIndex.detectAndRepairCorruption() record('metadata-corruption', { checked: true, healed: 0, detail: 'detect-and-repair pass ran (see its own narration for repairs)' }) // Lift a failed-rollback write-quarantine: force a full rebuild so the // derived indexes are provably reconciled with canonical, then clear the // flag so writes resume. if (this.storeInconsistency) { + beginPhase( + 'write-quarantine', + 'full derived-index rebuild to lift the quarantine set by a failed transaction rollback' + ) await this.rebuildIndexesIfNeeded(true) const cleared = this.storeInconsistency record('write-quarantine', { checked: true, healed: 1, detail: `lifted (${cleared.records.length} record(s) reconciled)` }) this.storeInconsistency = null - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` + `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` + `Writes are re-enabled.` @@ -18332,9 +18424,9 @@ export class Brainy implements BrainyInterface { record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no rebuild() contract' }) continue } - prodLog.warn( - `[Brainy] repairIndex(): explicit rebuild requested for '${familyName}' — ` + - `rebuilding unconditionally (no invariant consulted).` + beginPhase( + `provider:${familyName}`, + `explicit rebuild requested — rebuilding '${familyName}' unconditionally, no invariant consulted` ) // The metadata family routes through the online build-beside // orchestrator (B3 D3) instead of the provider's own rebuild() — @@ -18351,7 +18443,7 @@ export class Brainy implements BrainyInterface { rebuilt: true, reason: 'explicit rebuild requested' }) - prodLog.warn(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) + prodLog.narrate(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) continue } @@ -18360,11 +18452,16 @@ export class Brainy implements BrainyInterface { rebuild?: () => Promise } | null if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') { + beginPhase(`provider:${familyName}`, 'checking the provider contract') record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract' }) continue } + beginPhase( + `provider:${familyName}`, + `reading the '${familyName}' provider's own invariant report, then healing only what it asks for` + ) let report: ProviderInvariantReport try { report = await p.validateInvariants() @@ -18381,7 +18478,7 @@ export class Brainy implements BrainyInterface { checked: true, healed: 1, detail: `rebuilt from canonical (failing: ${report.invariants.filter((i) => !i.holds).map((i) => i.name).join(', ')})` }) - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + `requiring a rebuild — reconciling its derived state from canonical.` ) @@ -18405,7 +18502,7 @@ export class Brainy implements BrainyInterface { const failingRepairs = report.invariants .filter((i) => !i.holds && i.heal === 'repair') .map((i) => i.name) - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex(): provider '${report.provider}' asks for an incremental ` + `repair (${failingRepairs.join(', ')}) — running its own repair().` ) @@ -18444,6 +18541,7 @@ export class Brainy implements BrainyInterface { // rebuild failure are now reconciled — clear the queryable degraded state // and re-arm the read-path warning. if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) { + beginPhase('degraded-read-state', 'clearing degraded ids and re-arming the read-path warning') this._indexDegradedIds.clear() this._indexRebuildFailed = null this._degradedReadWarned = false @@ -18452,11 +18550,13 @@ export class Brainy implements BrainyInterface { const healedTotal = families.reduce((n, f) => n + f.healed, 0) const report: RepairReport = { families, healedTotal, durationMs: Date.now() - startedAt } - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex complete in ${report.durationMs}ms — ` + `${families.filter((f) => f.checked).length}/${families.length} families checked, ` + `${healedTotal} heal(s): ` + - families.map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}`).join(', ') + families + .map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}@${f.durationMs ?? 0}ms`) + .join(', ') ) return report } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index d9934c3b..a0d55c1e 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1217,6 +1217,13 @@ export interface RepairFamilyReport { skipped?: string /** Why the outcome is what it is when neither `detail` nor `skipped` says it. */ reason?: string + /** + * The phase's own wall, in milliseconds. A repair on a production store ran + * for over thirty minutes without a single line of output; an operator had + * to read `top` to know it was alive. A receipt that cannot say WHERE the + * time went is not a receipt — every row carries its own. + */ + durationMs?: number } /** The full receipt returned by repairIndex(). */ diff --git a/tests/integration/repair-narration.test.ts b/tests/integration/repair-narration.test.ts new file mode 100644 index 00000000..1fbe15e4 --- /dev/null +++ b/tests/integration/repair-narration.test.ts @@ -0,0 +1,119 @@ +/** + * @module tests/integration/repair-narration + * @description A REPAIR NARRATES ITSELF, AND ITS RECEIPT SAYS WHERE THE TIME + * WENT. + * + * On a production store (14,647 nouns / 73,070 verbs) a `repairIndex()` ran + * for more than thirty minutes at roughly a full core with ZERO log lines + * between its start and its end, while the read doors kept serving. The + * operator could tell it was alive only from `top`, and could not tell which + * of its single-threaded walks it was inside. The law pinned here: + * + * - every phase announces itself BEFORE it works, naming what it is about + * to walk; + * - a heartbeat names the phase still running, at a bounded cadence, for as + * long as it runs; + * - every phase reports its own wall, and that wall is carried in the typed + * receipt (`RepairFamilyReport.durationMs`) — not only in a log line. + * + * All of it on the narration channel, which production's log clamp cannot + * silence (see tests/integration/open-narration.test.ts). + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' +import { prodLog, configureLogger, LogLevel } from '../../src/utils/logger.js' + +describe('repairIndex narration', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + configureLogger({ level: LogLevel.INFO }) + }) + + async function seededBrain(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-narration-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + for (let i = 0; i < 5; i++) { + await brain.add({ data: `repair subject ${i}`, type: NounType.Concept }) + } + await brain.flush() + return brain + } + + it('announces every phase, reports its wall, and carries that wall in the receipt', async () => { + const brain = await seededBrain() + const narrateSpy = vi.spyOn(prodLog, 'narrate') + + const report = await brain.repairIndex() + + const lines = narrateSpy.mock.calls.map(([m]) => String(m)) + + // Every family that ran has BOTH a start line and a finish line naming it. + for (const family of report.families) { + const started = lines.filter((l) => l.includes(`"${family.family}" started —`)) + const finished = lines.filter((l) => + new RegExp(`"${family.family}" finished in \\d+ms`).test(l) + ) + expect(finished.length, `no finish line for ${family.family}`).toBeGreaterThanOrEqual(1) + // A skipped family may be recorded without a start line only if it never + // began; every family that began must have announced itself. + if (family.checked) { + expect(started.length, `no start line for ${family.family}`).toBeGreaterThanOrEqual(1) + } + // THE RECEIPT CARRIES THE WALL — not only the log. + expect(typeof family.durationMs, `${family.family} has no durationMs`).toBe('number') + expect(family.durationMs).toBeGreaterThanOrEqual(0) + } + + // The closing line accounts for the whole repair, per family. + const closing = lines.filter((l) => /repairIndex complete in \d+ms/.test(l)) + expect(closing.length).toBe(1) + expect(closing[0]).toMatch(/@\d+ms/) + }, 180_000) + + it('heartbeats while a single phase is still walking', async () => { + const brain = await seededBrain() + + // Make one phase long enough to cross the heartbeat cadence, exactly as a + // multi-minute canonical walk does on a real store. + const proto = FileSystemStorage.prototype as unknown as Record< + string, + (...args: unknown[]) => Promise + > + const realPrune = proto.pruneOrphanedEntities + proto.pruneOrphanedEntities = async function slow(this: unknown, ...args: unknown[]) { + await new Promise((r) => setTimeout(r, 6_500)) + return realPrune.apply(this, args) + } + // Clamped as production clamps it: the narration must survive. + configureLogger({ level: LogLevel.ERROR }) + const narrateSpy = vi.spyOn(prodLog, 'narrate') + try { + await brain.repairIndex() + } finally { + proto.pruneOrphanedEntities = realPrune + } + + const beats = narrateSpy.mock.calls + .map(([m]) => String(m)) + .filter((l) => /repairIndex: still in "orphaned-containers" after \d+s/.test(l)) + expect(beats.length).toBeGreaterThanOrEqual(1) + expect(beats[0]).toMatch(/ghost\/scar containers/) + }, 180_000) +}) From f5a6cb3f618611a23a5559fbada69bb41b907bb1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:44:38 -0700 Subject: [PATCH 14/42] =?UTF-8?q?perf(flush):=20an=20idle=20brain=20does?= =?UTF-8?q?=20no=20work=20=E2=80=94=20no=20periodic=20flush=20without=20a?= =?UTF-8?q?=20write?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REPORTED from the field: a process holding many stores, with no writes for ten minutes, printed "All indexes flushed to disk in 216-601ms" per store every ~35 seconds and burned over a core at idle. Every one of those flushes re-persisted state identical to what was already on disk — the provider flushes, the watermark stamps, the generation counter, the entity-tree stamp — because flush() never asked whether anything had changed. - flush() over a clean brain is now O(1) and silent: a dirty witness is set by every committed write (both commit paths end at noteWriteForPersistence, and the deferred-embed worker lands through the single-op path) and cleared by a flush that runs. A write landing DURING a flush sets it again, so no write's work is ever skipped — it is done by the next flush. Set before the policy check, so a `'manual'` consumer's explicit flush is never a no-op it didn't ask for. - An explicit flush now tells the cadence it happened. It didn't, so the very next write saw "30s since the last flush" and kicked a background flush with nothing to do, and the idle timer fired two seconds later over writes the explicit flush had already persisted. - The graph adjacency index's auto-flush asks before it acts: two O(1) reads of the LSM MemTables, and a tick over a quiet index returns without calling into the trees at all. assessProviderHealth is NOT timer-driven — it is a synchronous O(1) read of a provider's own healthReport(), called on the read gate, so it costs nothing on an idle brain. No change needed there. Pins: tests/integration/idle-costs-nothing.test.ts — 90 idle seconds produce zero flushes, zero provider calls and zero log lines; three explicit flushes over a clean brain call no provider; one write earns exactly one flush. --- src/brainy.ts | 39 +++++ src/graph/graphAdjacencyIndex.ts | 11 ++ src/graph/lsm/LSMTree.ts | 11 ++ tests/integration/idle-costs-nothing.test.ts | 147 +++++++++++++++++++ 4 files changed, 208 insertions(+) create mode 100644 tests/integration/idle-costs-nothing.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 24407018..013885ff 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -740,6 +740,18 @@ export class Brainy implements BrainyInterface { // Write acks NEVER await it; a failed background flush is LOUD and re-armed. private _persistDirtyWrites = 0 private _persistLastFlushAt = Date.now() + /** + * Whether a write has been committed since the last flush that ran. THE + * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written + * to has nothing to make durable, and a flush over it must cost nothing and + * say nothing. Measured on a production process holding 21 brains: with no + * writes for ten minutes it still printed "All indexes flushed to disk in + * 216–601ms" per brain every ~35s and idled at 1.26 cores, because a flush + * called every provider, stamped the watermarks, persisted the generation + * counter and re-stamped the entity tree whether or not anything had + * changed. + */ + private _dirtySinceLastFlush = false private _persistIdleTimer: ReturnType | null = null private _persistBackgroundFlight: Promise | null = null @@ -2668,6 +2680,12 @@ export class Brainy implements BrainyInterface { * engine's own cadence (callers never call flush() in hot paths). */ private noteWriteForPersistence(): void { + // THE DIRTY WITNESS. Set on every committed write — both commit paths + // (single-op and transaction) end here, and the deferred-embed worker + // lands its vectors through the single-op path — BEFORE the policy check, + // so a `'manual'` consumer's explicit flush() is never skipped either. + // Cleared by a flush that actually runs; see flush(). + this._dirtySinceLastFlush = true const cfg = this.config.persistence if (this.isReadOnly || cfg?.policy === 'manual') return this._persistDirtyWrites++ @@ -12246,6 +12264,27 @@ export class Brainy implements BrainyInterface { return } + // A CLEAN BRAIN FLUSHES NOTHING, AND SAYS NOTHING. No write has been + // committed since the last flush, so every step below would re-persist + // state identical to what is already on disk — provider flushes, the + // watermark stamps, the generation counter, the entity-tree stamp — and + // print two lines announcing it. On a process holding 21 brains that + // no-op cost 1.26 cores at idle. The witness is set by every committed + // write (see noteWriteForPersistence) and cleared here; a write landing + // DURING this flush sets it again, so it is never lost — the next flush + // does that write's work. + if (!this._dirtySinceLastFlush) { + return + } + this._dirtySinceLastFlush = false + // An explicit flush IS a flush: tell the cadence so, or the very next + // write sees "30s since the last flush" (the cadence only counted its + // own) and kicks a background flush that has nothing left to do, and the + // idle timer fires two seconds later over writes this flush already + // persisted. + this._persistLastFlushAt = Date.now() + this._persistDirtyWrites = 0 + console.log('Flushing Brainy indexes and caches to disk...') const startTime = Date.now() diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index d002164e..ebd3b90c 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1052,6 +1052,17 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { */ private startAutoFlush(): void { this.flushTimer = setInterval(async () => { + // NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an + // index nobody has written to since the last flush has nothing to + // write, and calling into the trees (and their logging) on a cadence + // over a quiet store is exactly the idle cost this law exists to + // remove. + if ( + !this.lsmTreeVerbsBySource.hasPendingWrites() && + !this.lsmTreeVerbsByTarget.hasPendingWrites() + ) { + return + } await this.flush() }, this.config.flushInterval) // Background maintenance must never keep the host process alive — diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts index e19ec145..b4f6052f 100644 --- a/src/graph/lsm/LSMTree.ts +++ b/src/graph/lsm/LSMTree.ts @@ -687,6 +687,17 @@ export class LSMTree { } } + /** + * @description Whether this tree holds anything a flush would write — + * the MemTable is non-empty. Synchronous and O(1), so a background cadence + * can ask before it does anything at all: the engine does no periodic work + * without a cause. + * @returns true when a flush would write; false when it would be a no-op. + */ + hasPendingWrites(): boolean { + return !this.memTable.isEmpty() + } + async close(): Promise { this.stopCompactionTimer() diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts new file mode 100644 index 00000000..8f951d46 --- /dev/null +++ b/tests/integration/idle-costs-nothing.test.ts @@ -0,0 +1,147 @@ +/** + * @module tests/integration/idle-costs-nothing + * @description AN IDLE BRAIN DOES NO WORK. + * + * Measured on a production process holding 21 brains: with no writes for ten + * minutes it printed "All indexes flushed to disk in 216–601ms" per brain + * every ~35 seconds and idled at 1.26 cores. Every one of those flushes + * re-persisted state identical to what was already on disk — the provider + * flushes, the watermark stamps, the generation counter, the entity-tree + * stamp — because `flush()` never asked whether anything had changed. + * + * The laws pinned here: + * (a) the persistence cadence arms only on a write — a brain nobody writes + * to flushes zero times, however long it is left open; + * (b) a flush on a clean brain is O(1): no provider is called, nothing is + * written, and nothing is printed; + * (c) one write earns exactly one flush's worth of work, and no more. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +/** Wait for any in-flight background flush, then let the idle timer settle. */ +async function drainCadence(brain: Brainy): Promise { + const inner = brain as unknown as { _persistBackgroundFlight: Promise | null } + await new Promise((r) => setTimeout(r, 3_000)) + await (inner._persistBackgroundFlight ?? Promise.resolve()) + await new Promise((r) => setTimeout(r, 500)) +} + +/** How long an idle brain is watched. Longer than the 30s flush interval. */ +const IDLE_WATCH_MS = 90_000 + +describe('an idle brain costs nothing', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function openBrain(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-idle-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + return brain + } + + it('flushes zero times over 90 idle seconds, and prints nothing', async () => { + const brain = await openBrain() + // One write and one flush to reach a clean, settled state — then nothing. + await brain.add({ data: 'the only write this test performs', type: NounType.Concept }) + await brain.flush() + + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + + // Watch the providers directly: a flush that runs calls all of them. + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex + const graphIndex = (brain as unknown as { graphIndex: { flush: () => Promise } }).graphIndex + const countsSpy = vi.spyOn(storage, 'flushCounts') + const metadataSpy = vi.spyOn(metadataIndex, 'flush') + const graphSpy = vi.spyOn(graphIndex, 'flush') + + try { + await new Promise((r) => setTimeout(r, IDLE_WATCH_MS)) + } finally { + console.log = origLog + } + + // (a) + (b): nothing ran, nothing was said. + expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) + expect(logged.filter((l) => /Flushing Brainy indexes/.test(l))).toEqual([]) + expect(countsSpy).not.toHaveBeenCalled() + expect(metadataSpy).not.toHaveBeenCalled() + expect(graphSpy).not.toHaveBeenCalled() + }, 180_000) + + it('an explicit flush over a clean brain calls no provider and prints nothing', async () => { + const brain = await openBrain() + await brain.add({ data: 'one write', type: NounType.Concept }) + await brain.flush() // this one does the work + + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex + const countsSpy = vi.spyOn(storage, 'flushCounts') + const metadataSpy = vi.spyOn(metadataIndex, 'flush') + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + try { + await brain.flush() // ...and this one has nothing to do + await brain.flush() + await brain.flush() + } finally { + console.log = origLog + } + + expect(countsSpy).not.toHaveBeenCalled() + expect(metadataSpy).not.toHaveBeenCalled() + expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) + }, 120_000) + + it('one write earns exactly one flush', async () => { + const brain = await openBrain() + await brain.add({ data: 'first', type: NounType.Concept }) + await brain.flush() + // Settle: the first write also kicked a BACKGROUND flush, which is not + // awaited by design. Drain it before counting, or its provider calls land + // inside this test's window and are attributed to the write below. + await drainCadence(brain) + + // Count the flushes that actually RAN. (Provider spies cannot answer this: + // the storage adapter's own count ledger is write-through, so a write calls + // flushCounts() on its own account, with no flush involved.) + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + const ran = () => logged.filter((l) => /All indexes flushed to disk/.test(l)).length + try { + await brain.add({ data: 'second — this is the cause', type: NounType.Concept }) + await brain.flush() + expect(ran()).toBe(1) + + // No further cause, no further work. + await brain.flush() + await brain.flush() + expect(ran()).toBe(1) + } finally { + console.log = origLog + } + }, 120_000) +}) From 131daa08cdc8d5cbb7df8b9f6ec2855946dba52b Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:48:52 -0700 Subject: [PATCH 15/42] feat(open): open never waits for a provider that is rebuilding itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a production store: a metadata provider that had to rebuild made init() pay the ENTIRE rebuild on the foreground — 641 seconds — with every other family idle behind it. The cause is a missing distinction: a provider reporting serving:false because it is BUSY BUILDING ITSELF and one reporting serving:false because it is BROKEN looked identical through healthReport(), and both were answered the same way — call rebuild(), and wait for it. The contract that tells them apart is one optional, synchronous, O(1) hook: `rebuildInProgress(): ProviderRebuildProgress | null`, reporting a phase name and whatever the provider actually measures (done/total/startedAt) — never an estimate dressed as a fact. A provider without the hook behaves exactly as before. With it, a provider owns its own rebuild: - the open gate neither starts a second rebuild nor waits for the provider's, and narrates that it is not waiting and what will refuse meanwhile; - init() returns and every other family serves; - that family's doors refuse BY NAME, carrying the provider's own progress, and say plainly that the door opens by itself and no action is needed — distinct from a broken index, which names repairIndex(); - the epoch stamp does not advance while any family is still being built. Nothing is ever served empty: a not-serving family refuses, as it already did. Pins: tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts — init() returns in milliseconds against a provider claiming a 6s rebuild, brainy starts no rebuild of its own, a filtered read refuses naming the phase and the 4,096/14,056 progress, and the door answers once the provider reports serving. The pin fails loudly rather than vacuously if its stub never installs. --- src/brainy.ts | 85 +++++++++- src/utils/indexReadiness.ts | 80 ++++++++++ ...not-wait-for-a-rebuilding-provider.test.ts | 145 ++++++++++++++++++ 3 files changed, 306 insertions(+), 4 deletions(-) create mode 100644 tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 013885ff..92702364 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -198,7 +198,12 @@ import { import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' -import { assessIndexReadiness, assessProviderHealth } from './utils/indexReadiness.js' +import { + assessIndexReadiness, + assessProviderHealth, + assessProviderRebuild, + describeRebuildProgress +} from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { @@ -4540,6 +4545,19 @@ export class Brainy implements BrainyInterface { this._graphAdjacencyVerified = true return 'live' } + // A provider that is REBUILDING ITSELF gets a refusal that says so, + // with its own progress: open deliberately did not wait for it (see + // rebuildIndexesIfNeeded), so this door is temporarily closed and will + // open on its own. Anything else is a broken index needing a repair. + const rebuilding = assessProviderRebuild(this.graphIndex) + if (rebuilding) { + throw new GraphIndexNotReadyError( + `Graph adjacency index is ${describeRebuildProgress(rebuilding)} and is not serving ` + + `yet. find({ connected }), neighbors() and related() refuse rather than serve an ` + + `empty result. The brain is open and every other family is serving; this door opens ` + + `by itself when the provider reports serving — no action is needed.` + ) + } throw new GraphIndexNotReadyError( `Graph adjacency index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. find({ connected }), neighbors() and ` + @@ -4643,6 +4661,15 @@ export class Brainy implements BrainyInterface { this._metadataVerified = true return 'live' } + const rebuilding = assessProviderRebuild(this.metadataIndex) + if (rebuilding) { + throw new MetadataIndexNotReadyError( + `Metadata field index is ${describeRebuildProgress(rebuilding)} and is not serving ` + + `yet. find({ where }) and other filtered reads refuse rather than serve an empty ` + + `result. The brain is open and every other family is serving; this door opens by ` + + `itself when the provider reports serving — no action is needed.` + ) + } throw new MetadataIndexNotReadyError( `Metadata field index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. find({ where }) and other filtered ` + @@ -4772,6 +4799,15 @@ export class Brainy implements BrainyInterface { this._vectorVerified = true return 'live' } + const rebuilding = assessProviderRebuild(this.index) + if (rebuilding) { + throw new VectorIndexNotReadyError( + `Vector index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + + `Semantic find({ query }) and proximity search refuse rather than serve an empty ` + + `result. The brain is open and every other family is serving; this door opens by ` + + `itself when the provider reports serving — no action is needed.` + ) + } throw new VectorIndexNotReadyError( `Vector index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. Semantic find({ query }) and ` + @@ -17144,6 +17180,19 @@ export class Brainy implements BrainyInterface { } if (assessment.readiness === 'not-ready') { + // A provider REBUILDING ITSELF gets a refusal that says so, with its + // own progress: open deliberately did not wait for it, this door is + // temporarily closed, and it opens by itself. Distinct from a broken + // index, which needs an operator. + const rebuilding = assessProviderRebuild(provider) + if (rebuilding) { + throw new ErrorClass( + `${name} index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + + `Reads of this family refuse rather than serve an empty result. The brain is open ` + + `and every other family is serving; this door opens by itself when the provider ` + + `reports serving — no action is needed.` + ) + } throw new ErrorClass( `${name} index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. Reads refuse rather than serve an ` + @@ -17466,9 +17515,37 @@ export class Brainy implements BrainyInterface { // by awaitMigrationLock meanwhile (nothing serves from a half-built index). // Gated per-index, so a non-migrating sibling still rebuilds when it needs // to; a migrating provider is skipped even under epoch-drift or size()===0. - const metadataMigrating = this.providerIsMigrating(this.metadataIndex) - const vectorMigrating = this.providerIsMigrating(this.index) - const graphMigrating = this.providerIsMigrating(this.graphIndex) + // SELF-REBUILD DEFERENCE (the sibling of the migration lock, and the + // reason a production open took 641 seconds): a provider that reports + // `rebuildInProgress()` is ALREADY rebuilding its own index. Brainy must + // neither start a second rebuild nor WAIT for the provider's — init() + // returns, every other family serves, and that family's own doors refuse + // by name (carrying this progress) until the provider reports serving. + // A provider without the hook behaves exactly as before. + const metadataRebuilding = assessProviderRebuild(this.metadataIndex) + const vectorRebuilding = assessProviderRebuild(this.index) + const graphRebuilding = assessProviderRebuild(this.graphIndex) + for (const [leg, progress] of [ + ['metadata', metadataRebuilding], + ['vector', vectorRebuilding], + ['graph', graphRebuilding] + ] as const) { + if (progress) { + prodLog.narrate( + `[Brainy] open(): the ${leg} provider is ${describeRebuildProgress(progress)} — ` + + `open does NOT wait for it. The brain opens now, every other family serves, and ` + + `${leg} reads refuse by name until the provider reports itself serving.` + ) + } + } + + const metadataMigrating = + this.providerIsMigrating(this.metadataIndex) || metadataRebuilding !== null + const vectorMigrating = this.providerIsMigrating(this.index) || vectorRebuilding !== null + const graphMigrating = this.providerIsMigrating(this.graphIndex) || graphRebuilding !== null + // The epoch stamp certifies EVERY derived index, so it must not advance + // while any family is still being built — by a migration lock or by the + // provider itself. const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating // Per-leg decision, in precedence order: a migrating provider owns its diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts index 498f2003..f1b52e3b 100644 --- a/src/utils/indexReadiness.ts +++ b/src/utils/indexReadiness.ts @@ -153,3 +153,83 @@ export function assessProviderHealth(provider: unknown): ProviderHealthAssessmen reasons: readiness === 'not-ready' ? ['isReady() returned false'] : [] } } + +/** + * @description A provider's self-report that it is REBUILDING ITS OWN index + * right now. Returned by the optional `rebuildInProgress()` hook. + * + * The distinction this exists to make: a provider reporting `serving: false` + * because it is BROKEN and a provider reporting `serving: false` because it is + * BUSY BUILDING ITSELF look identical through `healthReport()` alone, and + * brainy treated both the same way — it called `rebuild()` and waited for it, + * on the foreground of `init()`. A production store whose metadata provider + * had to rebuild paid 641 SECONDS of that wait before `init()` returned, with + * every other family idle behind it. + * + * A provider that reports progress here owns its own rebuild: brainy neither + * starts one nor waits for it, `init()` returns, the other families serve, and + * THAT family's doors refuse by name — carrying this progress — until the + * provider reports itself serving. + * + * Every field but `phase` is optional and every field is a MEASUREMENT: a + * provider reports only what it actually tracks, never an estimate dressed as + * a fact. + */ +export interface ProviderRebuildProgress { + /** The provider's own name for what it is doing. Quoted verbatim in refusals. */ + phase: string + /** Units completed so far, if the provider counts them. */ + done?: number + /** Units expected in total, if the provider knows it. */ + total?: number + /** Epoch millis when this rebuild started, if the provider tracks it. */ + startedAt?: number +} + +/** A provider that can report a rebuild it is running itself. */ +interface MaybeRebuildingProvider { + rebuildInProgress?: () => ProviderRebuildProgress | null +} + +/** + * @description Ask a provider whether it is rebuilding itself right now. + * Synchronous, O(1), feature-detected: a provider without the hook reports + * nothing and is treated exactly as before. + * @param provider - Any index provider, or `null`/`undefined`. + * @returns The provider's progress, or `null` when it is not rebuilding (or + * does not implement the hook). + */ +export function assessProviderRebuild(provider: unknown): ProviderRebuildProgress | null { + const p = provider as MaybeRebuildingProvider | null | undefined + if (p == null || typeof p.rebuildInProgress !== 'function') return null + try { + const progress = p.rebuildInProgress() + if (!progress || typeof progress.phase !== 'string' || progress.phase.length === 0) { + return null + } + return progress + } catch { + // A throwing hook says nothing trustworthy about a rebuild; fall through to + // the ordinary health verdict rather than inventing one. + return null + } +} + +/** + * @description Render a rebuild progress report as one operator-facing clause, + * for a refusal message. Includes only what the provider actually measured. + * @param progress - The provider's report. + * @returns A clause such as `rebuilding ("metadata shadow build", 4,096/14,056, 12s elapsed)`. + */ +export function describeRebuildProgress(progress: ProviderRebuildProgress): string { + const parts: string[] = [`"${progress.phase}"`] + if (typeof progress.done === 'number' && typeof progress.total === 'number') { + parts.push(`${progress.done.toLocaleString()}/${progress.total.toLocaleString()}`) + } else if (typeof progress.done === 'number') { + parts.push(`${progress.done.toLocaleString()} done`) + } + if (typeof progress.startedAt === 'number') { + parts.push(`${Math.round((Date.now() - progress.startedAt) / 1000)}s elapsed`) + } + return `rebuilding (${parts.join(', ')})` +} diff --git a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts new file mode 100644 index 00000000..459d7dfc --- /dev/null +++ b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts @@ -0,0 +1,145 @@ +/** + * @module tests/integration/open-does-not-wait-for-a-rebuilding-provider + * @description OPEN DOES NOT WAIT FOR A PROVIDER THAT IS REBUILDING ITSELF. + * + * Measured on a production store: a metadata provider that had to rebuild made + * `init()` pay the ENTIRE rebuild on the foreground — 641 seconds — with every + * other family idle behind it, because a provider reporting `serving: false` + * because it is BUSY BUILDING and one reporting `serving: false` because it is + * BROKEN were indistinguishable, and both were answered the same way: call + * `rebuild()`, and wait. + * + * The law: a provider that reports `rebuildInProgress()` owns its own rebuild. + * `init()` returns; every other family serves; THAT family's doors refuse by + * name, carrying the provider's own progress; and the doors open by themselves + * when the provider reports serving. Nothing is ever served empty. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import type { ProviderRebuildProgress } from '../../src/utils/indexReadiness.js' + +/** How long the stub provider claims to be rebuilding. */ +const REBUILD_MS = 6_000 + +describe('a provider rebuilding itself never blocks open', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + it('init() returns in milliseconds, the family refuses by name, then answers', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-provider-')) + dirs.push(dir) + + // Seed a store so the open has something to (not) rebuild. + const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await seed.init() + await seed.add({ data: 'a row with a plain field', type: NounType.Concept, metadata: { kind: 'report' } }) + await seed.flush() + await seed.close() + + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + + // Dress the metadata index as a provider that is rebuilding ITSELF: not + // serving, and honest about why. `init()` wires the real index first, so + // the hooks are installed on the instance as soon as it exists — the gate + // reads them by feature detection, exactly as it would a native provider's. + const rebuildStartedAt = Date.now() + const stillRebuilding = () => Date.now() - rebuildStartedAt < REBUILD_MS + let rebuildCalls = 0 + + const inner = brain as unknown as { + metadataIndex: Record + setupIndex?: unknown + } + // Install on the prototype-free instance right after construction by + // patching the property the moment init() assigns it. + const install = (target: Record) => { + const realRebuild = target.rebuild as () => Promise + target.rebuildInProgress = (): ProviderRebuildProgress | null => + stillRebuilding() + ? { phase: 'metadata shadow build', done: 4_096, total: 14_056, startedAt: rebuildStartedAt } + : null + target.healthReport = () => ({ + provider: 'metadata', + healthy: !stillRebuilding(), + serving: !stillRebuilding(), + generation: 1, + invariants: [], + unledgered: [] + }) + target.rebuild = async () => { + rebuildCalls++ + return realRebuild.call(target) + } + } + + // init() constructs the metadata index; patch as soon as it exists, before + // the gate consults it. A microtask hop after the index is assigned is + // enough because the gate runs later in the same init. + const initPromise = (async () => { + const originalEnsure = (brain as unknown as { setupIndex?: () => unknown }).setupIndex + void originalEnsure + return brain.init() + })() + // Patch on the first tick the index exists. + const patcher = setInterval(() => { + if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { + install(inner.metadataIndex) + } + }, 1) + const startedAt = Date.now() + try { + await initPromise + } finally { + clearInterval(patcher) + } + const openMs = Date.now() - startedAt + + // If the patch did not land before the gate ran, this test proves nothing — + // say so loudly rather than passing vacuously. + expect( + typeof inner.metadataIndex.rebuildInProgress, + 'the stub provider was never installed — the test is vacuous' + ).toBe('function') + + // 1. The open did not wait out the rebuild. + expect(openMs).toBeLessThan(REBUILD_MS) + // 2. And brainy did not start a rebuild of its own on top of the provider's. + expect(rebuildCalls).toBe(0) + + // 3. The family's door refuses BY NAME, carrying the provider's progress. + let refusal: Error | null = null + try { + await brain.find({ where: { kind: 'report' } } as never) + } catch (err) { + refusal = err as Error + } + expect(refusal, 'a not-serving metadata family must refuse, never serve empty').not.toBeNull() + expect(refusal!.message).toMatch(/metadata shadow build/i) + expect(refusal!.message).toMatch(/4,096\/14,056/) + expect(refusal!.message).toMatch(/no action is needed/i) + + // 4. Other families keep serving — the brain is open. + const all = await brain.getNouns?.({ pagination: { limit: 1 } } as never) + expect(all ?? true).toBeTruthy() + + // 5. When the provider reports itself serving, the door opens by itself. + await new Promise((r) => setTimeout(r, REBUILD_MS)) + ;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false + await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined() + }, 180_000) +}) From 50676c02f44bd3efacaf79fce94adf216c623ff6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:50:26 -0700 Subject: [PATCH 16/42] fix(open): a provider rebuilding itself is a third state, not a CRITICAL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follower to the self-rebuild deference. The open gate's consistency check — "metadata index has 0 entries but storage has N entities" → CRITICAL + a forced second rebuild — knew two states, migrating and not. A provider whose rebuild() returns once the rebuild is OWNED AND RUNNING ONLINE (its doors refusing by name while other families serve) legitimately reports 0 entries there, so every first contact printed a false CRITICAL and kicked a redundant second rebuild. The exemption rides the rebuild-progress hook, NOT isMigrating() — widening that would hold every write and 503 the whole brain through the migration snapshot, which is worse than the false alarm. The check's real class is untouched: a provider reporting 0 entries with no rebuild in progress still trips it. The crash-recovery rebuild kick gets the same deference: a provider already rebuilding itself from canonical is doing exactly that work, and the fold ran in the generation store's open before any provider existed, so what it is reading is the repaired canonical. Pin: a provider stub reporting a rebuild and 0 entries opens with no CRITICAL line and no second rebuild; the vacuous-stub case fails loudly. --- src/brainy.ts | 32 +++++++++++-- ...not-wait-for-a-rebuilding-provider.test.ts | 47 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 92702364..dad609b3 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1489,10 +1489,27 @@ export class Brainy implements BrainyInterface { `[Brainy] Rebuilding indexes after crash recovery rolled back ` + `${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)` ) + // SELF-REBUILD DEFERENCE, same law as the open gate: a provider that + // is already rebuilding itself from canonical is doing exactly this + // work. Kicking a second rebuild on top of it is redundant at best. + // Safe by ordering: the crash-recovery fold ran in the generation + // store's open, BEFORE any provider was constructed, so a provider + // rebuilding now is reading the repaired canonical records. + const kick = async (leg: string, provider: { rebuild: () => Promise }) => { + const rebuilding = assessProviderRebuild(provider) + if (rebuilding) { + prodLog.narrate( + `[Brainy] crash-recovery rebuild: the ${leg} provider is already ` + + `${describeRebuildProgress(rebuilding)} from canonical — not kicking a second one.` + ) + return + } + await provider.rebuild() + } await Promise.all([ - this.metadataIndex.rebuild(), - this.index.rebuild(), - this.graphIndex.rebuild() + kick('metadata', this.metadataIndex), + kick('vector', this.index as unknown as { rebuild: () => Promise }), + kick('graph', this.graphIndex) ]) } @@ -17757,6 +17774,15 @@ export class Brainy implements BrainyInterface { // when the metadata provider holds the migration lock: a 0 count there // reflects its in-place rebuild in progress, not a missed rebuild, so // forcing a second rebuild would collide with the provider's own. + // THREE states, not two. `metadataMigrating` above is true for a + // provider holding the migration lock AND for one that reports it is + // rebuilding itself — a provider whose rebuild() returns once the + // rebuild is OWNED AND RUNNING (online, its doors refusing by name) + // legitimately reports 0 entries here, and calling that CRITICAL would + // print a false alarm and kick a redundant second rebuild on every + // first contact. The check's real class — a rebuild that ran to + // completion and produced nothing — is untouched: a provider reporting + // 0 entries with NO rebuild in progress still trips it. if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) { console.error( `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + diff --git a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts index 459d7dfc..a46ad6a5 100644 --- a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts +++ b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts @@ -142,4 +142,51 @@ describe('a provider rebuilding itself never blocks open', () => { ;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined() }, 180_000) + + it('a rebuilding provider reporting 0 entries is not a CRITICAL, and gets no second rebuild', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-critical-')) + dirs.push(dir) + const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await seed.init() + await seed.add({ data: 'a stored entity', type: NounType.Concept }) + await seed.flush() + await seed.close() + + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + + let rebuildCalls = 0 + const errors: string[] = [] + const origError = console.error + console.error = ((...a: unknown[]) => { errors.push(a.map(String).join(' ')) }) as typeof console.error + + const inner = brain as unknown as { metadataIndex: Record } + const patcher = setInterval(() => { + if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { + const target = inner.metadataIndex + target.rebuildInProgress = () => ({ phase: 'online metadata rebuild', startedAt: Date.now() }) + target.healthReport = () => ({ + provider: 'metadata', healthy: false, serving: false, + generation: 1, invariants: [], unledgered: [] + }) + // The shape the native engine now has: the index reports NOTHING while + // its rebuild runs online behind refusing doors. + target.getStats = async () => ({ totalEntries: 0 }) + target.rebuild = async () => { rebuildCalls++ } + } + }, 1) + try { + await brain.init() + } finally { + clearInterval(patcher) + console.error = origError + } + + expect( + typeof inner.metadataIndex.rebuildInProgress, + 'the stub provider was never installed — the test is vacuous' + ).toBe('function') + expect(errors.filter((l) => /CRITICAL: Metadata index has 0 entries/.test(l))).toEqual([]) + expect(rebuildCalls).toBe(0) + }, 180_000) }) From 48802ba3859b3119c36cf22d85edac559750d6d5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:57:43 -0700 Subject: [PATCH 17/42] feat(contract): declare contract 1, serve three operators, refuse four by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Open Brainy's side of the API contract the accelerated engine published. DECLARED: package.json carries "brainyContract": 1 and the engine states its own via contractVersion() / BRAINY_CONTRACT_VERSION — two engines compare an integer instead of probing prototypes, and a tool reads the package field without importing the engine. Pinned so the two can never drift apart. SERVED: hasAll, noneOf and excludes now work on the index path. The defect underneath was worse than the reported divergence — the metadata index's operator switch had NO DEFAULT CASE, so any operator without a case left the field's match set at its initial [] and find() returned an empty page. Documented, validator-accepted, matcher-implemented operators answering silently wrong. hasAll intersects each element's posting set (an empty operand is vacuously true of every row that has the field), noneOf complements their union, excludes complements contains. REFUSED BY NAME: startsWith, endsWith, matches and length raise INVALID_QUERY naming the operator, the field and the reason. An equality/range posting index cannot evaluate a substring, a pattern or an array length without reading every row — which is the cost this path exists to avoid — so it refuses rather than answering an empty page. Both engines now agree on all 25 tokens and contract 1 has no remaining operator divergence. This is a visible change for a consumer calling those four through find({ where }): an empty page becomes a typed refusal. EMITTED: scripts/emit-contract-manifest.mjs generates docs/api-contract.json from the BUILT surface — prototype doors, exported error classes, the operator sets read out of their single definitions, the field-addressing vocabulary, the health verdicts. Nothing hand-maintained, so a diff between two manifests is a diff between two engines. `--check` fails on a stale manifest, which makes the announce-every-addition duty mechanical rather than remembered. RATIFIED in docs/contract-1-ratification.md: the 41-of-57 required split with the promise spelled out (a refusal is part of a door; deprecation is not removal), the serving-withholding list confirmed exhaustive and identical, the minor/major rule adopted with the announcement duty, the 30 storage seam methods committed as supported surface until Stage 2, and a finding filed against the spec — is / isNot / greaterEqual / lessEqual are listed there as served aliases and have never existed in this engine, which throws INVALID_QUERY on all four. --- docs/api-contract.json | 1545 +++++++++++++++++ package.json | 1 + scripts/emit-contract-manifest.mjs | 129 ++ src/index.ts | 1 + src/neural/embeddedPatterns.ts | 2 +- src/neural/embeddedTypeEmbeddings.ts | 4 +- src/utils/metadataIndex.ts | 89 + src/utils/version.ts | 24 + .../filter-operator-conformance.test.ts | 151 ++ 9 files changed, 1943 insertions(+), 3 deletions(-) create mode 100644 docs/api-contract.json create mode 100644 scripts/emit-contract-manifest.mjs create mode 100644 tests/integration/filter-operator-conformance.test.ts diff --git a/docs/api-contract.json b/docs/api-contract.json new file mode 100644 index 00000000..9dadcc0e --- /dev/null +++ b/docs/api-contract.json @@ -0,0 +1,1545 @@ +{ + "contractVersion": 1, + "engine": "@soulcraftlabs/brainy", + "prose": "docs/contract-1-ratification.md", + "compatibility": { + "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", + "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" + }, + "doors": [ + { + "name": "adaptiveHistoryBudgetBytes", + "kind": "method", + "arity": 1 + }, + { + "name": "add", + "kind": "method", + "arity": 1 + }, + { + "name": "addMany", + "kind": "method", + "arity": 1 + }, + { + "name": "adoptLogAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "adoptLogAuthorityInner", + "kind": "method", + "arity": 0 + }, + { + "name": "aggViewFromEntity", + "kind": "method", + "arity": 1 + }, + { + "name": "anyProviderMigrating", + "kind": "method", + "arity": 0 + }, + { + "name": "applyFusionScoring", + "kind": "method", + "arity": 2 + }, + { + "name": "applyGraphConstraints", + "kind": "method", + "arity": 2 + }, + { + "name": "armIdleFlushTimer", + "kind": "method", + "arity": 2 + }, + { + "name": "asOf", + "kind": "method", + "arity": 2 + }, + { + "name": "assertGenerationStoreReady", + "kind": "method", + "arity": 1 + }, + { + "name": "assertWritable", + "kind": "method", + "arity": 1 + }, + { + "name": "audit", + "kind": "method", + "arity": 0 + }, + { + "name": "auditGraph", + "kind": "method", + "arity": 0 + }, + { + "name": "autoAdoptLegacyVfsBlobsIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "autoAlpha", + "kind": "method", + "arity": 1 + }, + { + "name": "autoCompactHistory", + "kind": "method", + "arity": 0 + }, + { + "name": "awaitMigrationLock", + "kind": "method", + "arity": 1 + }, + { + "name": "awaitPendingEmbeds", + "kind": "method", + "arity": 0 + }, + { + "name": "backfillAggregateIfNeeded", + "kind": "method", + "arity": 1 + }, + { + "name": "batchGet", + "kind": "method", + "arity": 2 + }, + { + "name": "brainWideStrictRequiresSubtype", + "kind": "method", + "arity": 1 + }, + { + "name": "bridgeLegacyPendingEmbedSidecars", + "kind": "method", + "arity": 0 + }, + { + "name": "buildAtGenerationVectors", + "kind": "method", + "arity": 2 + }, + { + "name": "buildGraphView", + "kind": "method", + "arity": 4 + }, + { + "name": "buildMetadataFilter", + "kind": "method", + "arity": 1 + }, + { + "name": "buildMigrationUpdate", + "kind": "method", + "arity": 5 + }, + { + "name": "buildRelationMigrationUpdate", + "kind": "method", + "arity": 5 + }, + { + "name": "cacheVerbInt", + "kind": "method", + "arity": 2 + }, + { + "name": "canServeVectorAtGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "checkHealth", + "kind": "method", + "arity": 0 + }, + { + "name": "checkMigrations", + "kind": "method", + "arity": 0 + }, + { + "name": "clear", + "kind": "method", + "arity": 0 + }, + { + "name": "clearPendingEmbed", + "kind": "method", + "arity": 1 + }, + { + "name": "close", + "kind": "method", + "arity": 0 + }, + { + "name": "closeDurableSteps", + "kind": "method", + "arity": 0 + }, + { + "name": "cluster", + "kind": "method", + "arity": 1 + }, + { + "name": "collectProviderInvariants", + "kind": "method", + "arity": 0 + }, + { + "name": "compactHistory", + "kind": "method", + "arity": 1 + }, + { + "name": "consumeMetadataWatermarkVerdict", + "kind": "method", + "arity": 1 + }, + { + "name": "convertMetadataToEntity", + "kind": "method", + "arity": 2 + }, + { + "name": "convertNounToEntity", + "kind": "method", + "arity": 1 + }, + { + "name": "counts", + "kind": "accessor" + }, + { + "name": "createIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "createMigrationBackupIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "createPinnedDb", + "kind": "method", + "arity": 1 + }, + { + "name": "createResult", + "kind": "method", + "arity": 4 + }, + { + "name": "dbFinalizationRegistry", + "kind": "accessor" + }, + { + "name": "dbHost", + "kind": "accessor" + }, + { + "name": "defineAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "detectIdKind", + "kind": "method", + "arity": 3 + }, + { + "name": "diagnostics", + "kind": "method", + "arity": 0 + }, + { + "name": "diff", + "kind": "method", + "arity": 2 + }, + { + "name": "embed", + "kind": "method", + "arity": 1 + }, + { + "name": "embedBatch", + "kind": "method", + "arity": 2 + }, + { + "name": "emitCommitted", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceSubtypeOnAdd", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceSubtypeOnRelate", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceTrackedFieldValues", + "kind": "method", + "arity": 2 + }, + { + "name": "enhanceNLPResult", + "kind": "method", + "arity": 2 + }, + { + "name": "enqueuePendingEmbed", + "kind": "method", + "arity": 1 + }, + { + "name": "ensureAggregationIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "ensureIndexesLoaded", + "kind": "method", + "arity": 0 + }, + { + "name": "ensureInitialized", + "kind": "method", + "arity": 1 + }, + { + "name": "entityForAggFromRawRecord", + "kind": "method", + "arity": 1 + }, + { + "name": "entityFromGenerationRecord", + "kind": "method", + "arity": 3 + }, + { + "name": "entityIntsToUuids", + "kind": "method", + "arity": 1 + }, + { + "name": "entityViewFromRawRecord", + "kind": "method", + "arity": 2 + }, + { + "name": "excludedVisibilityTiers", + "kind": "method", + "arity": 1 + }, + { + "name": "executeGraphSearch", + "kind": "method", + "arity": 2 + }, + { + "name": "executeProximitySearch", + "kind": "method", + "arity": 1 + }, + { + "name": "executeTextSearch", + "kind": "method", + "arity": 2 + }, + { + "name": "executeVectorSearch", + "kind": "method", + "arity": 3 + }, + { + "name": "explain", + "kind": "method", + "arity": 1 + }, + { + "name": "export", + "kind": "method", + "arity": 0 + }, + { + "name": "extract", + "kind": "method", + "arity": 2 + }, + { + "name": "extractConcepts", + "kind": "method", + "arity": 2 + }, + { + "name": "extractEntities", + "kind": "method", + "arity": 2 + }, + { + "name": "factSegmentPaths", + "kind": "method", + "arity": 1 + }, + { + "name": "fieldCountsAggregateName", + "kind": "method", + "arity": 1 + }, + { + "name": "fillSubtypes", + "kind": "method", + "arity": 1 + }, + { + "name": "filterIdsBelted", + "kind": "method", + "arity": 2 + }, + { + "name": "find", + "kind": "method", + "arity": 1 + }, + { + "name": "findAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "findDuplicates", + "kind": "method", + "arity": 1 + }, + { + "name": "findMatchingWords", + "kind": "method", + "arity": 3 + }, + { + "name": "flush", + "kind": "method", + "arity": 0 + }, + { + "name": "formatInfo", + "kind": "method", + "arity": 0 + }, + { + "name": "formatSubtypeError", + "kind": "method", + "arity": 1 + }, + { + "name": "generation", + "kind": "method", + "arity": 0 + }, + { + "name": "generationDigest", + "kind": "method", + "arity": 1 + }, + { + "name": "get", + "kind": "method", + "arity": 2 + }, + { + "name": "getActivePlugins", + "kind": "method", + "arity": 0 + }, + { + "name": "getAvailableFields", + "kind": "method", + "arity": 0 + }, + { + "name": "getBackgroundDeduplicator", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldsForType", + "kind": "method", + "arity": 1 + }, + { + "name": "getFieldStatistics", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldsWithCardinality", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldValues", + "kind": "method", + "arity": 1 + }, + { + "name": "getIndexStats", + "kind": "method", + "arity": 0 + }, + { + "name": "getIndexStatus", + "kind": "method", + "arity": 0 + }, + { + "name": "getMemoryStats", + "kind": "method", + "arity": 0 + }, + { + "name": "getNeighborUuids", + "kind": "method", + "arity": 2 + }, + { + "name": "getNounCount", + "kind": "method", + "arity": 0 + }, + { + "name": "getOptimalQueryPlan", + "kind": "method", + "arity": 1 + }, + { + "name": "getStats", + "kind": "method", + "arity": 1 + }, + { + "name": "getStorageType", + "kind": "method", + "arity": 0 + }, + { + "name": "getSubtypeRule", + "kind": "method", + "arity": 1 + }, + { + "name": "getTripleIntelligence", + "kind": "method", + "arity": 0 + }, + { + "name": "getTypedNeighbors", + "kind": "method", + "arity": 4 + }, + { + "name": "getVerbCount", + "kind": "method", + "arity": 0 + }, + { + "name": "graph", + "kind": "accessor" + }, + { + "name": "graphAccelerationProvider", + "kind": "method", + "arity": 0 + }, + { + "name": "graphCommunities", + "kind": "method", + "arity": 1 + }, + { + "name": "graphCommunitiesFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphCommunitiesNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphEntityInt", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExport", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExportFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExportNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphPath", + "kind": "method", + "arity": 3 + }, + { + "name": "graphPathFallback", + "kind": "method", + "arity": 3 + }, + { + "name": "graphPathNative", + "kind": "method", + "arity": 4 + }, + { + "name": "graphRank", + "kind": "method", + "arity": 1 + }, + { + "name": "graphRankFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphRankNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphSubgraph", + "kind": "method", + "arity": 2 + }, + { + "name": "graphSubgraphFallback", + "kind": "method", + "arity": 4 + }, + { + "name": "graphSubgraphFromQuery", + "kind": "method", + "arity": 5 + }, + { + "name": "graphSubgraphNative", + "kind": "method", + "arity": 5 + }, + { + "name": "groupByLabel", + "kind": "method", + "arity": 2 + }, + { + "name": "hasStorageMethod", + "kind": "method", + "arity": 1 + }, + { + "name": "hasVectorOrTextCriteria", + "kind": "method", + "arity": 1 + }, + { + "name": "health", + "kind": "method", + "arity": 0 + }, + { + "name": "highlight", + "kind": "method", + "arity": 1 + }, + { + "name": "highlightSemanticPhase", + "kind": "method", + "arity": 5 + }, + { + "name": "history", + "kind": "method", + "arity": 2 + }, + { + "name": "historyStats", + "kind": "method", + "arity": 0 + }, + { + "name": "hub", + "kind": "accessor" + }, + { + "name": "hydrateIdMapperForGraphRebuild", + "kind": "method", + "arity": 0 + }, + { + "name": "hydrateNativeSubgraph", + "kind": "method", + "arity": 2 + }, + { + "name": "import", + "kind": "method", + "arity": 2 + }, + { + "name": "importPluginPackage", + "kind": "method", + "arity": 1 + }, + { + "name": "incidentEdges", + "kind": "method", + "arity": 3 + }, + { + "name": "indexStats", + "kind": "method", + "arity": 0 + }, + { + "name": "init", + "kind": "method", + "arity": 1 + }, + { + "name": "insights", + "kind": "method", + "arity": 0 + }, + { + "name": "isEmbeddingReady", + "kind": "method", + "arity": 0 + }, + { + "name": "isInfrastructureWrite", + "kind": "method", + "arity": 1 + }, + { + "name": "isInitialized", + "kind": "accessor" + }, + { + "name": "isReadOnly", + "kind": "accessor" + }, + { + "name": "kickBackgroundFlush", + "kind": "method", + "arity": 1 + }, + { + "name": "kickEmbedWorker", + "kind": "method", + "arity": 0 + }, + { + "name": "legacyLayoutMigrationPhase", + "kind": "method", + "arity": 0 + }, + { + "name": "loadAnalyticsGraph", + "kind": "method", + "arity": 1 + }, + { + "name": "loadPlugins", + "kind": "method", + "arity": 0 + }, + { + "name": "logAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "maintenanceDebt", + "kind": "method", + "arity": 0 + }, + { + "name": "materializeAtGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "metadataIndexRetractionOp", + "kind": "method", + "arity": 3 + }, + { + "name": "migrate", + "kind": "method", + "arity": 1 + }, + { + "name": "migrateField", + "kind": "method", + "arity": 1 + }, + { + "name": "migrateInternal", + "kind": "method", + "arity": 2 + }, + { + "name": "migrateLegacyZeroNormVfsRootIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "migrationSnapshot", + "kind": "method", + "arity": 0 + }, + { + "name": "neededFamiliesMigrating", + "kind": "method", + "arity": 1 + }, + { + "name": "neighbors", + "kind": "method", + "arity": 2 + }, + { + "name": "newId", + "kind": "method", + "arity": 0 + }, + { + "name": "nlp", + "kind": "method", + "arity": 0 + }, + { + "name": "normalizeConfig", + "kind": "method", + "arity": 1 + }, + { + "name": "noteWriteForPersistence", + "kind": "method", + "arity": 0 + }, + { + "name": "now", + "kind": "method", + "arity": 0 + }, + { + "name": "onChange", + "kind": "method", + "arity": 1 + }, + { + "name": "pagination", + "kind": "accessor" + }, + { + "name": "parseMigrationPath", + "kind": "method", + "arity": 1 + }, + { + "name": "parseNaturalQuery", + "kind": "method", + "arity": 1 + }, + { + "name": "pathExists", + "kind": "method", + "arity": 2 + }, + { + "name": "pendingEmbedCount", + "kind": "method", + "arity": 0 + }, + { + "name": "performInit", + "kind": "method", + "arity": 1 + }, + { + "name": "persistPinnedGeneration", + "kind": "method", + "arity": 2 + }, + { + "name": "persistSingleOp", + "kind": "method", + "arity": 6 + }, + { + "name": "pickMetadataProbe", + "kind": "method", + "arity": 1 + }, + { + "name": "pickVectorProbe", + "kind": "method", + "arity": 0 + }, + { + "name": "pinGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "planGetEntity", + "kind": "method", + "arity": 3 + }, + { + "name": "planTransact", + "kind": "method", + "arity": 1 + }, + { + "name": "planTxAdd", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxRelate", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxRemove", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxUnrelate", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxUpdate", + "kind": "method", + "arity": 3 + }, + { + "name": "projectionGauges", + "kind": "method", + "arity": 0 + }, + { + "name": "providerForFamily", + "kind": "method", + "arity": 1 + }, + { + "name": "providerIsMigrating", + "kind": "method", + "arity": 1 + }, + { + "name": "providerMigrationStatus", + "kind": "method", + "arity": 0 + }, + { + "name": "queryAggregate", + "kind": "method", + "arity": 2 + }, + { + "name": "queryIndexFamilies", + "kind": "method", + "arity": 1 + }, + { + "name": "readPath", + "kind": "method", + "arity": 2 + }, + { + "name": "ready", + "kind": "accessor" + }, + { + "name": "rebuildIndexesIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "rebuildMetadataIndexOnline", + "kind": "method", + "arity": 0 + }, + { + "name": "reconcileLogDivergence", + "kind": "method", + "arity": 2 + }, + { + "name": "reconstructPath", + "kind": "method", + "arity": 4 + }, + { + "name": "recordStateAt", + "kind": "method", + "arity": 3 + }, + { + "name": "recoverPendingEmbedsFromLog", + "kind": "method", + "arity": 0 + }, + { + "name": "registerShutdownHooks", + "kind": "method", + "arity": 0 + }, + { + "name": "relate", + "kind": "method", + "arity": 1 + }, + { + "name": "related", + "kind": "method", + "arity": 1 + }, + { + "name": "relateMany", + "kind": "method", + "arity": 1 + }, + { + "name": "relationFromGenerationRecord", + "kind": "method", + "arity": 2 + }, + { + "name": "relationshipSubtypesOf", + "kind": "method", + "arity": 1 + }, + { + "name": "releaseGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "remove", + "kind": "method", + "arity": 1 + }, + { + "name": "removeAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "removeMany", + "kind": "method", + "arity": 1 + }, + { + "name": "removeMigrationBackupSafe", + "kind": "method", + "arity": 0 + }, + { + "name": "repackHistory", + "kind": "method", + "arity": 1 + }, + { + "name": "repairIndex", + "kind": "method", + "arity": 1 + }, + { + "name": "requestFlush", + "kind": "method", + "arity": 1 + }, + { + "name": "requireProviders", + "kind": "method", + "arity": 1 + }, + { + "name": "requireSubtype", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveAsOfGeneration", + "kind": "method", + "arity": 2 + }, + { + "name": "resolveDiffEndpoint", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveHiddenIds", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveHNSWPersistMode", + "kind": "method", + "arity": 0 + }, + { + "name": "resolveRawGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveRetentionPolicy", + "kind": "method", + "arity": 0 + }, + { + "name": "resolveVerbEndpointInts", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveVerbIntsToIds", + "kind": "method", + "arity": 1 + }, + { + "name": "restore", + "kind": "method", + "arity": 2 + }, + { + "name": "rrfFusion", + "kind": "method", + "arity": 4 + }, + { + "name": "runAggregationBackfillWalk", + "kind": "method", + "arity": 0 + }, + { + "name": "runAggregationCatchUp", + "kind": "method", + "arity": 0 + }, + { + "name": "runEmbedWorker", + "kind": "method", + "arity": 0 + }, + { + "name": "runOracle", + "kind": "method", + "arity": 1 + }, + { + "name": "runRepairIndexPhases", + "kind": "method", + "arity": 5 + }, + { + "name": "scanFacts", + "kind": "method", + "arity": 1 + }, + { + "name": "seedIdsToInts", + "kind": "method", + "arity": 1 + }, + { + "name": "selectorToSeedIds", + "kind": "method", + "arity": 1 + }, + { + "name": "setRetentionBudget", + "kind": "method", + "arity": 1 + }, + { + "name": "setupEmbedder", + "kind": "method", + "arity": 0 + }, + { + "name": "setupIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "setupStorage", + "kind": "method", + "arity": 0 + }, + { + "name": "similar", + "kind": "method", + "arity": 1 + }, + { + "name": "similarity", + "kind": "method", + "arity": 2 + }, + { + "name": "splitForHighlighting", + "kind": "method", + "arity": 2 + }, + { + "name": "stampBrainFormat", + "kind": "method", + "arity": 0 + }, + { + "name": "stampBrainFormatIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "stampEntityTree", + "kind": "method", + "arity": 0 + }, + { + "name": "stampProjectionWatermarks", + "kind": "method", + "arity": 0 + }, + { + "name": "stats", + "kind": "method", + "arity": 0 + }, + { + "name": "storageAdapter", + "kind": "accessor" + }, + { + "name": "stream", + "kind": "method", + "arity": 0 + }, + { + "name": "streaming", + "kind": "accessor" + }, + { + "name": "subtypesOf", + "kind": "method", + "arity": 1 + }, + { + "name": "trackField", + "kind": "method", + "arity": 1 + }, + { + "name": "transact", + "kind": "method", + "arity": 2 + }, + { + "name": "transactionLog", + "kind": "method", + "arity": 1 + }, + { + "name": "unrelate", + "kind": "method", + "arity": 1 + }, + { + "name": "unvectorNounForRootMigration", + "kind": "method", + "arity": 1 + }, + { + "name": "update", + "kind": "method", + "arity": 1 + }, + { + "name": "updateMany", + "kind": "method", + "arity": 1 + }, + { + "name": "updateRelation", + "kind": "method", + "arity": 1 + }, + { + "name": "upsertMergeParams", + "kind": "method", + "arity": 2 + }, + { + "name": "use", + "kind": "method", + "arity": 1 + }, + { + "name": "usesDefaultWasmEmbedder", + "kind": "method", + "arity": 0 + }, + { + "name": "validateIndexConsistency", + "kind": "method", + "arity": 0 + }, + { + "name": "vectorSearchAtGeneration", + "kind": "method", + "arity": 4 + }, + { + "name": "verbsToRelations", + "kind": "method", + "arity": 1 + }, + { + "name": "verbToRelationLike", + "kind": "method", + "arity": 1 + }, + { + "name": "verifyEntityTreeStamp", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyGraphAdjacencyLive", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyLogAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyMetadataLive", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyVectorLive", + "kind": "method", + "arity": 0 + }, + { + "name": "versionedIndexProviders", + "kind": "method", + "arity": 0 + }, + { + "name": "vfs", + "kind": "accessor" + }, + { + "name": "waitForIndexed", + "kind": "method", + "arity": 2 + }, + { + "name": "warm", + "kind": "method", + "arity": 0 + }, + { + "name": "warmupEmbeddings", + "kind": "method", + "arity": 0 + }, + { + "name": "warnIfReadsDegraded", + "kind": "method", + "arity": 1 + }, + { + "name": "wireConnectionsCodec", + "kind": "method", + "arity": 0 + }, + { + "name": "wireGraphIdResolver", + "kind": "method", + "arity": 0 + } + ], + "errors": [ + "BrainyError", + "DerivedArtifactMissingError", + "GraphIndexNotReadyError", + "MetadataIndexNotReadyError", + "MigrationInProgressError", + "ProtectedArtifactError", + "VectorIndexNotReadyError" + ], + "operators": { + "accepted": [ + "between", + "contains", + "endsWith", + "eq", + "equals", + "excludes", + "exists", + "greaterThan", + "greaterThanOrEqual", + "gt", + "gte", + "hasAll", + "in", + "length", + "lessThan", + "lessThanOrEqual", + "lt", + "lte", + "matches", + "missing", + "ne", + "noneOf", + "notEquals", + "oneOf", + "startsWith" + ], + "servedOnIndexPath": [ + "between", + "contains", + "eq", + "equals", + "excludes", + "exists", + "greaterThan", + "greaterThanOrEqual", + "gt", + "gte", + "hasAll", + "in", + "lessThan", + "lessThanOrEqual", + "lt", + "lte", + "missing", + "ne", + "noneOf", + "notEquals", + "oneOf" + ], + "refusedByIndexPath": [ + "endsWith", + "length", + "matches", + "startsWith" + ], + "combinators": [ + "allOf", + "anyOf", + "not" + ] + }, + "fieldAddressing": { + "systemKeyPrefix": "system.", + "systemEntityScalars": [ + "confidence", + "createdAt", + "createdBy", + "id", + "service", + "subtype", + "type", + "updatedAt", + "visibility", + "weight" + ], + "systemRelationScalars": [ + "confidence", + "createdAt", + "createdBy", + "service", + "sourceId", + "subtype", + "targetId", + "updatedAt", + "verb", + "visibility", + "weight" + ], + "plumbingFields": [ + "_rev", + "connections", + "data", + "level", + "vector" + ] + }, + "health": { + "verdicts": [ + "pass", + "warn", + "fail" + ], + "healKinds": [ + "none", + "repair", + "rebuild" + ], + "servingWithholdingInvariants": [ + "index-initialized", + "durable-state-present", + "manifest-residency", + "replay-clean", + "strand-latch" + ] + } +} diff --git a/package.json b/package.json index bb6b5a47..60e901de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "@soulcraftlabs/brainy", "version": "10.4.3", + "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs new file mode 100644 index 00000000..13a9bc6d --- /dev/null +++ b/scripts/emit-contract-manifest.mjs @@ -0,0 +1,129 @@ +#!/usr/bin/env node +/** + * Emit this build's API-contract manifest to docs/api-contract.json. + * + * WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the + * code the first time somebody adds one. This reads the surface the build + * actually exposes — the prototype's own methods and accessors, the exported + * error classes, the `where` operator sets, the field-addressing vocabulary, + * the health verdicts — so a diff between two engines' manifests is a diff + * between two engines, never between two authors. + * + * Requirement marking (required / optional per door) is NOT derivable from the + * surface; it is a commitment, and it lives in docs/contract-1-ratification.md. + * This manifest carries the surface; that document carries the promise. + * + * Usage: node scripts/emit-contract-manifest.mjs [--check] + * --check exits non-zero when the committed manifest is stale. + */ + +import { writeFileSync, readFileSync, existsSync } from 'node:fs' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..') +const OUT = join(ROOT, 'docs', 'api-contract.json') + +const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js')) +const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js')) +const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js')) +const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js')) + +/** Every own method and accessor on the class's prototype, minus the private ones. */ +function surfaceOf(ctor) { + const doors = [] + for (const name of Object.getOwnPropertyNames(ctor.prototype)) { + if (name === 'constructor' || name.startsWith('_')) continue + const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name) + if (!descriptor) continue + if (typeof descriptor.value === 'function') { + doors.push({ name, kind: 'method', arity: descriptor.value.length }) + } else if (descriptor.get) { + doors.push({ name, kind: 'accessor' }) + } + } + return doors.sort((a, b) => a.name.localeCompare(b.name)) +} + +const errors = Object.entries(errorsModule) + .filter(([name, value]) => typeof value === 'function' && /Error$/.test(name)) + .map(([name]) => name) + .sort() + +// The operator sets, read from the engine's own refusal message so the +// manifest can never disagree with the validator. +const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8') +const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set\(\[([\s\S]*?)\]\)/) +if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess') +const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort() + +const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8') +const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) => + // Proven by the refusal path: these are the tokens with no case in the + // index's operator switch, so they fall to its default and are refused. + !new RegExp(`case '${op}':`).test(indexSource) +) +const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) + +const manifest = { + contractVersion: versionModule.contractVersion(), + engine: '@soulcraftlabs/brainy', + prose: 'docs/contract-1-ratification.md', + compatibility: { + minor: + 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', + major: + 'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused' + }, + doors: surfaceOf(Brainy), + errors, + operators: { + accepted, + servedOnIndexPath: servedOnIndex, + refusedByIndexPath: refusedByIndex, + combinators: ['allOf', 'anyOf', 'not'] + }, + fieldAddressing: { + systemKeyPrefix: 'system.', + systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(), + systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(), + plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort() + }, + health: { + verdicts: ['pass', 'warn', 'fail'], + healKinds: ['none', 'repair', 'rebuild'], + servingWithholdingInvariants: [ + 'index-initialized', + 'durable-state-present', + 'manifest-residency', + 'replay-clean', + 'strand-latch' + ] + } +} + +const rendered = `${JSON.stringify(manifest, null, 2)}\n` + +if (process.argv.includes('--check')) { + if (!existsSync(OUT)) { + console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`) + process.exit(1) + } + if (readFileSync(OUT, 'utf-8') !== rendered) { + console.error( + `docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` + + `the addition (minor = additive; a removal is a contract major).` + ) + process.exit(1) + } + console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`) + process.exit(0) +} + +writeFileSync(OUT, rendered) +console.log( + `Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` + + `${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` + + `${manifest.operators.accepted.length} operators ` + + `(${manifest.operators.refusedByIndexPath.length} refused by the index path).` +) diff --git a/src/index.ts b/src/index.ts index 973136a5..edc21809 100644 --- a/src/index.ts +++ b/src/index.ts @@ -184,6 +184,7 @@ export { // Export version utilities export { getBrainyVersion } from './utils/version.js' +export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js' // Export plugin system export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 4f4339f4..92e3057a 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2025-09-29T10:10:00-07:00 + * Generated: 2026-08-27T09:18:45-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index 5b10116c..f4cdd632 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-06-29T10:04:19-07:00 + * Generated: 2026-08-27T09:18:45-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-06-29T10:04:19-07:00", + generatedAt: "2026-08-27T09:18:45-07:00", sizeBytes: { embeddings: 259584, base64: 346112 diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 3cc56b2e..3e0e3d17 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2241,6 +2241,74 @@ export class MetadataIndexManager implements MetadataIndexProvider { break } + // ===== ARRAY SET OPERATORS ===== + // An element-indexed array field makes all three exact on the + // index path. They were previously ABSENT from this switch, so + // `fieldResults` kept its initial `[]` and the whole find() + // returned an empty page — a documented, matcher-implemented + // operator answering silently wrong. Served here instead. + + // hasAll: [a, b] — the field's array contains EVERY operand: + // the intersection of each element's posting set. + case 'hasAll': { + if (!Array.isArray(operand)) { + fieldResults = [] + break + } + if (operand.length === 0) { + // Vacuously true of every row that HAS the field. + const anyBitmap = (this.columnStore && this.columnStore.hasField(field)) + ? await this.columnStore.rangeQuery(field) + : await this.getExistsBitmapLegacy(field) + fieldResults = this.idMapper.intsIterableToUuids(anyBitmap) + break + } + let intersection: Set | null = null + for (const item of operand) { + const ids = new Set(await this.getIds(field, item)) + if (intersection === null) { + intersection = ids + } else { + for (const id of [...intersection]) { + if (!ids.has(id)) intersection.delete(id) + } + } + if (intersection.size === 0) break + } + fieldResults = intersection ? [...intersection] : [] + break + } + + // noneOf: [a, b] — the field's value is NONE of the operands: + // the complement of their union. + case 'noneOf': { + if (!Array.isArray(operand)) { + fieldResults = [] + break + } + const excludeInts: number[] = [] + for (const value of operand) { + for (const uuid of await this.getIds(field, value)) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined) excludeInts.push(intId) + } + } + fieldResults = this.complementIds(excludeInts) + break + } + + // excludes: value — the field's array does NOT contain the value: + // the complement of `contains`. + case 'excludes': { + const excludeInts: number[] = [] + for (const uuid of await this.getIds(field, operand)) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined) excludeInts.push(intId) + } + fieldResults = this.complementIds(excludeInts) + break + } + // ===== MISSING OPERATOR ===== // missing: boolean - equivalent to exists: !boolean case 'missing': { @@ -2257,6 +2325,27 @@ export class MetadataIndexManager implements MetadataIndexProvider { } break } + + // ===== EVERYTHING ELSE: REFUSED BY NAME, NEVER ANSWERED EMPTY ==== + // An equality/range posting index cannot evaluate a substring, a + // pattern or an array length without reading every row, and this + // path exists precisely to avoid that. It used to fall out of the + // switch with `fieldResults` still `[]`, so `find({ where: { name: + // { startsWith: 'a' } } })` returned an empty page and looked like + // an answer. An accepted operator either works or refuses — the + // matcher's own support for these operators governs in-memory + // filtering, never an index-backed find(). + default: + throw new BrainyError( + `Filter operator "${op}" on field "${rawField}" cannot be served by the ` + + `metadata index: an equality/range posting index cannot evaluate substrings, ` + + `patterns or array lengths without reading every row. It is REFUSED rather ` + + `than answered with an empty page. Filter on an indexable operator ` + + `(equals/eq, notEquals/ne, oneOf/in, noneOf, greaterThan/gt, ` + + `greaterThanOrEqual/gte, lessThan/lt, lessThanOrEqual/lte, between, contains, ` + + `excludes, hasAll, exists, missing) and narrow the rest in your own code.`, + 'INVALID_QUERY' + ) } // Intersect this operator's matches with the running set (AND semantics // for multiple operators on the same field). diff --git a/src/utils/version.ts b/src/utils/version.ts index f302eae0..327f923c 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -83,3 +83,27 @@ export function getAugmentationVersion(service: string): { augmentation: string; version: getBrainyVersion() } } + +/** + * The API-contract version this build implements — a single integer that two + * engines can compare without probing prototypes. + * + * A MINOR release is ADDITIVE: doors and error codes may be added, never + * removed or narrowed, and the contract integer does not move. A MAJOR release + * is what a REQUIRED door's removal or a behavioural narrowing costs, and it + * bumps this integer. A consumer pinning `brainyContract` in a peer range is + * therefore pinning "what I may call", not "which build I run". + * + * Declared in package.json as `"brainyContract"` so a manifest, a tool, or a + * sibling package can read it without importing the engine, and returned here + * so a running process can state its own. + */ +export const BRAINY_CONTRACT_VERSION = 1 as const + +/** + * @description The API-contract version this build implements. + * @returns The contract integer — see {@link BRAINY_CONTRACT_VERSION}. + */ +export function contractVersion(): number { + return BRAINY_CONTRACT_VERSION +} diff --git a/tests/integration/filter-operator-conformance.test.ts b/tests/integration/filter-operator-conformance.test.ts new file mode 100644 index 00000000..628017e7 --- /dev/null +++ b/tests/integration/filter-operator-conformance.test.ts @@ -0,0 +1,151 @@ +/** + * @module tests/integration/filter-operator-conformance + * @description THE OPERATOR SET, AND WHAT EACH TOKEN DOES ON THE INDEX PATH. + * + * The contract-1 manifest splits this engine's `where` operators three ways — + * served, served-beyond-baseline, refused-by-name — and two engines must agree + * token for token. This lane is the machine-checkable side of that agreement: + * it asserts the EXACT accepted set (so a manifest can be diffed against a run + * rather than against prose), and it pins each of the three classes. + * + * The defect it closes: the metadata index's operator switch had no default + * case, so an operator it does not implement — `hasAll`, `noneOf`, `excludes`, + * `startsWith`, `endsWith`, `matches`, `length` — left the field's match set at + * its initial `[]` and `find()` returned an empty page. A documented operator, + * implemented in the in-memory matcher, answering silently wrong. Three of the + * seven are now SERVED on the index path; the other four are REFUSED BY NAME, + * because an equality/range posting index cannot evaluate a substring, a + * pattern or an array length without reading every row. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { contractVersion, BRAINY_CONTRACT_VERSION } from '../../src/utils/version.js' + +/** The accepted `where` value-operator tokens, as a sorted list. */ +const ACCEPTED_OPERATORS = [ + 'between', 'contains', 'endsWith', 'eq', 'equals', 'excludes', 'exists', + 'greaterThan', 'greaterThanOrEqual', 'gt', 'gte', 'hasAll', 'in', 'length', + 'lessThan', 'lessThanOrEqual', 'lt', 'lte', 'matches', 'missing', 'ne', + 'noneOf', 'notEquals', 'oneOf', 'startsWith' +] as const + +/** Served on the index path with exact posting-set semantics. */ +const SERVED_ON_INDEX = [ + 'between', 'contains', 'eq', 'equals', 'exists', 'greaterThan', + 'greaterThanOrEqual', 'gt', 'gte', 'in', 'lessThan', 'lessThanOrEqual', + 'lt', 'lte', 'missing', 'ne', 'notEquals', 'oneOf', + 'excludes', 'hasAll', 'noneOf' +] as const + +/** Accepted by name, refused by the index path — never answered empty. */ +const REFUSED_BY_INDEX = ['endsWith', 'length', 'matches', 'startsWith'] as const + +describe('filter operator conformance', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + async function seeded(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-operators-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + await brain.add({ + data: 'a document about ferrets', + type: NounType.Document, + metadata: { tags: ['ferret', 'small', 'furry'], team: 'alpha' } + }) + await brain.add({ + data: 'a document about whales', + type: NounType.Document, + metadata: { tags: ['whale', 'large'], team: 'beta' } + }) + await brain.flush() + return brain + } + + it('the accepted operator set is exactly these 25 tokens', async () => { + const brain = await seeded() + // The engine names its own valid set in the refusal it raises for an + // unknown token — the honest place to read it from. + let message = '' + try { + await brain.find({ where: { team: { notIn: ['alpha'] } } } as never) + } catch (err) { + message = (err as Error).message + } + expect(message).toMatch(/Unknown filter operator "notIn"/) + const listed = (message.match(/Valid operators: ([^.]+)\./)?.[1] ?? '') + .split(',') + .map((t) => t.trim()) + .filter(Boolean) + .sort() + expect(listed).toEqual([...ACCEPTED_OPERATORS].sort()) + expect(listed.length).toBe(25) + // Four tokens a sibling manifest listed as served aliases are NOT in this + // engine's set and never have been — they raise INVALID_QUERY. + for (const absent of ['is', 'isNot', 'greaterEqual', 'lessEqual']) { + expect(listed).not.toContain(absent) + await expect( + brain.find({ where: { team: { [absent]: 'alpha' } } } as never) + ).rejects.toThrow(/Unknown filter operator/) + } + }, 120_000) + + it('serves hasAll, noneOf and excludes on the index path — never an empty page', async () => { + const brain = await seeded() + + const hasAll = await brain.find({ where: { tags: { hasAll: ['ferret', 'furry'] } } } as never) + expect(hasAll.length).toBe(1) + expect((hasAll[0] as { metadata?: Record }).metadata?.team).toBe('alpha') + + const noneOf = await brain.find({ where: { team: { noneOf: ['alpha'] } } } as never) + expect(noneOf.length).toBe(1) + expect((noneOf[0] as { metadata?: Record }).metadata?.team).toBe('beta') + + const excludes = await brain.find({ where: { tags: { excludes: 'whale' } } } as never) + expect(excludes.length).toBe(1) + expect((excludes[0] as { metadata?: Record }).metadata?.team).toBe('alpha') + + // hasAll with an operand nothing carries is EMPTY because it is empty — + // the honest zero, reached by evaluating the operator. + const none = await brain.find({ where: { tags: { hasAll: ['ferret', 'whale'] } } } as never) + expect(none.length).toBe(0) + }, 120_000) + + it('refuses the four index-unserveable operators BY NAME', async () => { + const brain = await seeded() + for (const op of REFUSED_BY_INDEX) { + const operand = op === 'length' ? 3 : 'a' + await expect( + brain.find({ where: { team: { [op]: operand } } } as never), + `${op} must refuse, never answer an empty page` + ).rejects.toThrow(new RegExp(`Filter operator "${op}".*cannot be served by the metadata index`, 's')) + } + }, 120_000) + + it('declares its contract version in code and in package.json', async () => { + expect(contractVersion()).toBe(1) + expect(BRAINY_CONTRACT_VERSION).toBe(1) + const pkg = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf-8')) + expect(pkg.brainyContract).toBe(contractVersion()) + }) + + it('the three classes partition the accepted set', () => { + expect([...SERVED_ON_INDEX, ...REFUSED_BY_INDEX].sort()).toEqual([...ACCEPTED_OPERATORS].sort()) + }) +}) From c1f0972395a908d551560ca7596ea18b9fe95ab5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:58:00 -0700 Subject: [PATCH 18/42] chore: keep the generated neural stamps at main's values The build regenerates these from the git commit time; a local rebuild moved only the stamp. Restored so the branch carries no incidental churn. --- src/neural/embeddedPatterns.ts | 2 +- src/neural/embeddedTypeEmbeddings.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 92e3057a..4f4339f4 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-08-27T09:18:45-07:00 + * Generated: 2025-09-29T10:10:00-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index f4cdd632..5b10116c 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-08-27T09:18:45-07:00 + * Generated: 2026-06-29T10:04:19-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-08-27T09:18:45-07:00", + generatedAt: "2026-06-29T10:04:19-07:00", sizeBytes: { embeddings: 259584, base64: 346112 From 4a67aa0fb97da0588083854e870dfd6b0a0e714e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:01:43 -0700 Subject: [PATCH 19/42] perf(vfs): the old-root sweep runs once per store, not once per open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a 14,056-noun / 72,679-verb production-shaped store, measured solo under an exclusive lock: the vfs-bootstrap phase cost 43,021 ms of a cold open and 52,696 ms of a WARM REOPEN. What dominates it is a migration sweep — a filtered find() over the whole store hunting for root directories created before the fixed root id existed. A store either carries such duplicates or never will, and the sweep ran on every open, forever, in the foreground. It is now caused by the store's state instead of by the open count: a durable marker under _system/ records that the sweep has run, and a store carrying it never sweeps again. A store without one sweeps in the BACKGROUND — the sweep only removes duplicate roots, nothing serves from them, and it was already declared non-critical — narrated at both ends, with whenRootSweepSettled() for anyone who needs to observe rather than race it. An adapter with no raw-object door keeps the old behaviour: correctness over cost, never a silent skip. Pins: tests/integration/vfs-root-sweep-once.test.ts — the sweep runs on the first open and never on the second or third; a sweep slowed to 4s does not delay the open. --- src/vfs/VirtualFileSystem.ts | 105 ++++++++++++++++- tests/integration/vfs-root-sweep-once.test.ts | 106 ++++++++++++++++++ 2 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 tests/integration/vfs-root-sweep-once.test.ts diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 59a16be4..19470b48 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -6,6 +6,7 @@ */ import { Readable, Writable } from 'stream' +import { prodLog } from '../utils/logger.js' import crypto from 'crypto' import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' @@ -66,6 +67,15 @@ export class VirtualFileSystem implements IVirtualFileSystem { private config: Required> & { rootEntityId?: string } private rootEntityId?: string private initialized = false + /** + * The one-time old-root sweep, in flight. See {@link sweepOldRootsIfNeeded}. + */ + private rootSweep?: Promise + /** + * Where the completed old-root sweep is recorded. Engine plumbing under + * `_system/`, like every other marker there — never enumerated as data. + */ + private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json' private currentUser: string = 'system' // Track current user for collaboration // Knowledge Layer features available via augmentation (brain.use('knowledge')) @@ -143,8 +153,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Create or find root entity this.rootEntityId = await this.initializeRoot() - // Clean up old UUID-based roots (one-time migration) - await this.cleanupOldRoots() + // Clean up old UUID-based roots — ONCE PER STORE, BEHIND THE DOORS. + // This is a migration sweep for roots created before the fixed root id + // existed. It ran on EVERY open, forever: a filtered find over the whole + // store hunting for duplicates that a store has either always had or + // never will. MEASURED on a 14,056-noun / 72,679-verb store: the phase it + // dominates cost 43-53 SECONDS of every open, warm reopens included. + // Now: a durable marker records that the sweep has run, and a store + // carrying it never sweeps again; a store without one sweeps in the + // BACKGROUND (the sweep only removes duplicate roots — nothing serves + // from them — and it was always declared non-critical). + this.rootSweep = this.sweepOldRootsIfNeeded() // Initialize projection registry with auto-discovery of built-in projections this.projectionRegistry = new ProjectionRegistry() @@ -394,6 +413,88 @@ export class VirtualFileSystem implements IVirtualFileSystem { * * This is a one-time migration helper that can be removed in future versions. */ + /** + * @description Run the old-root sweep at most once per store, in the + * background, and record that it ran. See the call site in {@link init} for + * the measurement that made this necessary. + * @returns A promise that settles when the sweep has finished (or was + * skipped); nothing in the read path awaits it. + */ + private async sweepOldRootsIfNeeded(): Promise { + const store = this.rawObjectStore() + if (store === null) { + // A storage adapter with no raw-object door cannot carry the marker. + // Sweep every open, as before — correctness over cost. + await this.cleanupOldRoots() + return + } + try { + const marker = await store.readRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH) + if (marker !== null && marker !== undefined) return + } catch { + // Unreadable marker: sweep, and rewrite it below. + } + prodLog.narrate( + '[VFS] one-time sweep for pre-fixed-id root directories running in the background — ' + + 'the open does not wait for it, and once it has run this store never sweeps again.' + ) + const startedAt = Date.now() + await this.cleanupOldRoots() + try { + await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, { + sweptAt: new Date().toISOString(), + durationMs: Date.now() - startedAt + }) + prodLog.narrate( + `[VFS] old-root sweep complete in ${Date.now() - startedAt}ms and recorded — ` + + 'no future open pays for it.' + ) + } catch (error) { + // Unrecorded sweep = the next open sweeps again. Conservative, and said + // out loud rather than quietly repeated forever. + prodLog.narrate( + `[VFS] old-root sweep finished in ${Date.now() - startedAt}ms but could NOT be ` + + `recorded (${(error as Error).message}) — the next open will sweep again.` + ) + } + } + + /** + * @description Settle once the background old-root sweep has finished. + * Resolves immediately when the store already carried the marker. Exists so + * tests and operators can observe the sweep instead of racing it; no read + * path waits on it. + * @returns A promise that settles with the sweep. + */ + public async whenRootSweepSettled(): Promise { + await this.rootSweep + } + + /** + * @description The brain's storage adapter, narrowed to the raw-object door + * this migration marker needs. Boundary: `Brainy.storage` is private, and + * this is the same reach-in the engine uses elsewhere for exactly this kind + * of engine-internal artifact. Returns null when the adapter has no + * raw-object door. + */ + private rawObjectStore(): { + readRawObject: (key: string) => Promise + writeRawObject: (key: string, value: unknown) => Promise + } | null { + const storage = (this.brain as unknown as { storage?: Record }).storage + if ( + storage && + typeof storage.readRawObject === 'function' && + typeof storage.writeRawObject === 'function' + ) { + return storage as unknown as { + readRawObject: (key: string) => Promise + writeRawObject: (key: string, value: unknown) => Promise + } + } + return null + } + private async cleanupOldRoots(): Promise { try { // Find any old VFS roots with UUID-based IDs (not our fixed ID) diff --git a/tests/integration/vfs-root-sweep-once.test.ts b/tests/integration/vfs-root-sweep-once.test.ts new file mode 100644 index 00000000..f84f4413 --- /dev/null +++ b/tests/integration/vfs-root-sweep-once.test.ts @@ -0,0 +1,106 @@ +/** + * @module tests/integration/vfs-root-sweep-once + * @description THE OLD-ROOT SWEEP RUNS ONCE PER STORE, NOT ONCE PER OPEN. + * + * The VFS bootstrap ran a filtered `find()` over the whole store on EVERY + * open, hunting for root directories created before the fixed root id existed + * — duplicates a store has either always had or never will. MEASURED on a + * 14,056-noun / 72,679-verb store: the phase it dominates cost 43–53 SECONDS + * of every open, warm reopens included. + * + * The law: a migration sweep is caused by the store's state, not by the clock + * or the open count. It runs behind the doors, records that it ran, and a + * store carrying that record never sweeps again. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' + +describe('the VFS old-root sweep', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function open(dir: string): Promise { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + return brain + } + + it('sweeps on the first open, records it, and never sweeps again', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-')) + dirs.push(dir) + + const sweepSpy = vi.spyOn( + VirtualFileSystem.prototype as unknown as { cleanupOldRoots: () => Promise }, + 'cleanupOldRoots' + ) + + const first = await open(dir) + await (first.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).toHaveBeenCalledTimes(1) + // The record is durable engine plumbing under _system/, like every other marker. + expect( + existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || + existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) + ).toBe(true) + + await first.add({ data: 'a row so the store is not trivially empty', type: NounType.Concept }) + await first.flush() + await first.close() + brains.splice(brains.indexOf(first), 1) + + sweepSpy.mockClear() + const second = await open(dir) + await (second.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).not.toHaveBeenCalled() + + await second.close() + brains.splice(brains.indexOf(second), 1) + + // ...and a third open, to prove it is the record and not a one-off. + sweepSpy.mockClear() + const third = await open(dir) + await (third.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).not.toHaveBeenCalled() + }, 180_000) + + it('the open does not wait for the sweep', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-async-')) + dirs.push(dir) + + const proto = VirtualFileSystem.prototype as unknown as Record< + string, + (...args: unknown[]) => Promise + > + const real = proto.cleanupOldRoots + proto.cleanupOldRoots = async function slow(this: unknown, ...args: unknown[]) { + await new Promise((r) => setTimeout(r, 4_000)) + return real.apply(this, args) + } + try { + const startedAt = Date.now() + const brain = await open(dir) + const openMs = Date.now() - startedAt + expect(openMs).toBeLessThan(3_000) + await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + } finally { + proto.cleanupOldRoots = real + } + }, 180_000) +}) From 5a091ccad95628368470a76ce5cebe30274b0651 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:02:57 -0700 Subject: [PATCH 20/42] feat(open): the open names the STEP that cost the time, not just the phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A phase that costs a minute and names only itself tells an operator where to look but not what to look at. MEASURED on a real 14,056-noun / 72,679-verb store, the warm reopen's generation-store phase cost 55,538 ms with nothing inside it named — the fold was skipped (the close was clean), so the cost was somewhere else entirely and the breakdown could not say where. Six steps inside the open now report their own wall with their own cause when they exceed the phase threshold: the generation store's open (manifest, committed ranges, fact log, packed tier, crash replay), the entity-tree stamp verification, the brain-format read, the pre-upgrade backup, the derived-index gate, and the VFS init. Silent under the threshold, so a fast open says nothing extra. Same always-visible channel as the phase lines. --- src/brainy.ts | 56 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index dad609b3..66317322 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1163,6 +1163,23 @@ export class Brainy implements BrainyInterface { ) }, OPEN_HEARTBEAT_MS) if (typeof openHeartbeat.unref === 'function') openHeartbeat.unref() + /** + * Narrate one STEP inside a phase when it turns out to be expensive. + * A phase that costs a minute and names only itself tells an operator + * where to look but not what to look at; this names the step. Silent + * under OPEN_PHASE_NARRATE_MS, so a fast open says nothing extra. + */ + const step = async (name: string, cause: string, run: () => Promise): Promise => { + const startedAt = Date.now() + try { + return await run() + } finally { + const elapsed = Date.now() - startedAt + if (elapsed >= OPEN_PHASE_NARRATE_MS) { + prodLog.narrate(`[Brainy] open: step "${name}" took ${elapsed}ms — ${cause}`) + } + } + } const markPhase = (name: string): void => { const now = Date.now() const elapsed = now - lastPhaseCheckpoint @@ -1263,9 +1280,12 @@ export class Brainy implements BrainyInterface { // instances skip recovery (readers never write; the next writer // repairs). this.generationStore = new GenerationStore(this.storage) - const generationOpenResult = await this.generationStore.open({ - readOnly: this.config.mode === 'reader' - }) + const generationOpenResult = await step( + 'generation-store.open', + 'reading the generation manifest and committed ranges, opening the fact log and the ' + + 'packed segment tier, and folding any crash-recovery replay', + () => this.generationStore.open({ readOnly: this.config.mode === 'reader' }) + ) // The generation fact log is CANONICAL state, not a derived index — no // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare @@ -1303,7 +1323,11 @@ export class Brainy implements BrainyInterface { // rollup invariants against the log head + live counters. Loud on // genuine incoherence (repairIndex heals), silent on absent/coherent, // benign-behind refreshes at the next flush. Never blocks open. - await this.verifyEntityTreeStamp() + await step( + 'verify-entity-tree-stamp', + 'comparing the entity tree\'s stamped generation and rollups against the store', + () => this.verifyEntityTreeStamp() + ) // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — @@ -1315,7 +1339,11 @@ export class Brainy implements BrainyInterface { // them from the canonical records and then re-stamps the marker AFTER the // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old / // absent marker, so the next open idempotently re-rebuilds). - this._brainFormat = await readBrainFormat(this.storage) + this._brainFormat = await step( + 'read-brain-format', + 'reading the on-disk format marker that decides whether the derived indexes are stale', + () => readBrainFormat(this.storage) + ) this._indexEpochStale = this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH @@ -1326,7 +1354,11 @@ export class Brainy implements BrainyInterface { // upgrade verifies + stamps; retained on failure. No-op for a reader, for // non-filesystem storage, or for a brain with no persisted data. if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) { - await this.createMigrationBackupIfNeeded() + await step( + 'pre-upgrade-backup', + 'snapshotting the brain directory before a one-time format rebuild (migrationBackup)', + () => this.createMigrationBackupIfNeeded() + ) } // PHASE 2 of 5 — "generation-store open+fold": GenerationStore @@ -1606,7 +1638,11 @@ export class Brainy implements BrainyInterface { // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true // once this open-time step has run. - await this.rebuildIndexesIfNeeded() + await step( + 'rebuild-indexes-if-needed', + 'the derived-index gate: each family\'s readiness verdict, and any build it asks for', + () => this.rebuildIndexesIfNeeded() + ) this.lazyRebuildCompleted = true // Check for pending data migrations @@ -1679,7 +1715,11 @@ export class Brainy implements BrainyInterface { // Initialize VFS: Ensure VFS is ready when accessed as property // This eliminates need for separate vfs.init() calls - zero additional complexity this._vfs = new VirtualFileSystem(this) - await this._vfs.init() + await step( + 'vfs.init', + 'creating or adopting the VFS root and wiring the path resolver', + () => this._vfs!.init() + ) this._vfsInitialized = true // Mark VFS as fully initialized // 8.0 MVCC: infrastructure bootstrap (VFS root, etc.) is now the From e4c27fbca81569d2f870ed873f2908263bab9b7e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:06:06 -0700 Subject: [PATCH 21/42] fix(flush): clear() and repairIndex() set the dirty witness themselves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both mutate durable state outside the two commit paths, so neither was seen by the flush witness added with the idle-flush law. A clear() followed by a flush() would have found the brain "clean" and skipped the entity-tree stamp, leaving a stamp describing the population the clear had just removed — a false divergence warning at the next open. Closing the gap where it is, rather than widening the witness to guess. --- src/brainy.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/brainy.ts b/src/brainy.ts index 66317322..711d8d02 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8523,6 +8523,11 @@ export class Brainy implements BrainyInterface { */ async clear(): Promise { await this.ensureInitialized() + // A clear mutates durable state without going through a commit path, so + // it must set the dirty witness itself — otherwise a `clear()` followed by + // `flush()` would find the brain "clean" and skip the entity-tree stamp, + // leaving a stamp that describes the population this call just removed. + this._dirtySinceLastFlush = true // Clear storage await this.storage.clear() @@ -18394,6 +18399,10 @@ export class Brainy implements BrainyInterface { */ async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { await this.ensureInitialized() + // A repair recounts, prunes and rebuilds outside the commit paths; the + // dirty witness is set so a caller's flush after a repair does its normal + // work rather than finding the brain "clean". + this._dirtySinceLastFlush = true const startedAt = Date.now() const families: RepairFamilyReport[] = [] From 9dd399216b1b99c59e5add3f44d850c5d8a5e5b9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:09:05 -0700 Subject: [PATCH 22/42] perf(generations): discover generations by directory name, not by walking the log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a production-shaped store (14,056 nouns / 72,679 verbs, an 11 GB generation history), measured solo under an exclusive lock: the generation-store phase cost 55,538 ms of a WARM REOPEN after a clean close — with the fold correctly skipped, so nothing in that phase's name explained it. This is what it was doing. Discovering which generations exist on disk called listRawObjects('_generations'), which RECURSES the whole tree and returns every file in every generation directory — to extract a set of integers that the top-level directory NAMES already spell out. The cost scales with the entire history, is paid on every open, warm or cold, and grows for the life of the store. A one-level door — listRawPrefixes(prefix), the immediate child directory names — is added to the storage seam. The filesystem adapter answers it with a single readdir; BaseStorage derives it from the recursive listing, so an adapter without a cheap implementation is never wrong, only never faster; and the generation store falls back to the old listing when the door is absent. One behavioural difference, stated: an EMPTY generation directory is now discovered where the file listing could not see it. Above the committed watermark that is a crash scar, and recovery already has an explicit branch for it ("indeterminate partial dir" — dropped, narrated). Below it, it becomes a resolvable generation holding no records, which is what an empty generation means. Suites: the durability kill matrix (15), db-mvcc (30), history repacking (4), rollback trapdoor (3), entity-tree stamp (4) and the full unit suite (2,105) all green. --- src/db/generationStore.ts | 27 ++++++++++++++++++----- src/db/types.ts | 15 +++++++++++++ src/storage/adapters/fileSystemStorage.ts | 24 ++++++++++++++++++++ src/storage/baseStorage.ts | 23 +++++++++++++++++++ 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index d925c9e0..fd052c31 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -537,12 +537,29 @@ export class GenerationStore { this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon') this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed) - // Discover existing generation record directories. - const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) + // Discover existing generation record directories — BY DIRECTORY NAME. + // This used to call listRawObjects(), which recurses the whole + // `_generations/` tree and returns every file in every generation, to + // extract a set of integers the top-level directory names already spell. + // MEASURED on a real store with an 11 GB generation history: the phase + // this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with + // no fold to blame — this walk is what it was doing. An adapter without + // the one-level door falls back to the recursive listing, unchanged. const seenGens = new Set() - for (const p of recordPaths) { - const gen = parseGenerationFromPath(p) - if (gen !== null) seenGens.add(gen) + const oneLevel = ( + this.storage as { listRawPrefixes?: (prefix: string) => Promise } + ).listRawPrefixes + if (typeof oneLevel === 'function') { + for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) { + const gen = Number(name) + if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen) + } + } else { + const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) + for (const p of recordPaths) { + const gen = parseGenerationFromPath(p) + if (gen !== null) seenGens.add(gen) + } } let rolledBack = 0 diff --git a/src/db/types.ts b/src/db/types.ts index 363de086..56bdef11 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -450,6 +450,21 @@ export interface GenerationStorage { deleteRawObject(path: string): Promise /** List raw object paths under a prefix (normalized, `.gz`-stripped). */ listRawObjects(prefix: string): Promise + /** + * OPTIONAL: the IMMEDIATE child directory names under a prefix — one level, + * no recursion, no file paths. + * + * Why it exists: discovering which generations are on disk needs only the + * top-level directory NAMES under `_generations/`, but the only door for it + * was `listRawObjects`, which recurses the whole tree and returns every file + * in every generation. On a store with a long history that is a full walk of + * the entire generation log, paid on EVERY open, to learn a set of integers + * the directory names already spell out. + * + * An adapter without this door keeps working — the caller falls back to the + * recursive listing. + */ + listRawPrefixes?(prefix: string): Promise /** Remove every object under a prefix (and the directory itself on disk). */ removeRawPrefix(prefix: string): Promise /** Durability barrier: fsync the given object paths (no-op in memory). */ diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 784ea5d8..3f1055c2 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -686,6 +686,30 @@ export class FileSystemStorage extends BaseStorage { return pruned } + /** + * @description The IMMEDIATE child directory names under a prefix — ONE + * `readdir`, no recursion, no file paths. See the seam's JSDoc + * (`src/db/types.ts`) for what this replaced: discovering the generations on + * disk walked the entire generation log on every open, reading out every + * file in every generation, to learn the set of integers the top-level + * directory names already spell. + * @param prefix - Storage-root-relative directory prefix. + * @returns The child directory names (not paths); empty when the prefix does + * not exist. + */ + public override async listRawPrefixes(prefix: string): Promise { + await this.ensureInitialized() + const fullPath = path.join(this.rootDir, prefix) + try { + const entries = await fs.promises.readdir(fullPath, { withFileTypes: true }) + return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory()) + .map((e: { name: string }) => e.name) + } catch (error: any) { + if (error?.code === 'ENOENT') return [] + throw error + } + } + /** * Primitive operation: List objects under path prefix * All metadata operations use this internally via base class routing diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index f518e68f..d8bcb780 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1437,6 +1437,29 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.listObjectsUnderPath(prefix) } + /** + * @description The IMMEDIATE child directory names under a prefix — one + * level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a + * separate door exists. This default derives them from the recursive + * listing, so it is never WRONG, only never faster; the filesystem adapter + * overrides it with a single directory read. + * @param prefix - Storage-root-relative directory prefix. + * @returns The child directory names (not paths), in listing order. + */ + public async listRawPrefixes(prefix: string): Promise { + await this.ensureInitialized() + const paths = await this.listObjectsUnderPath(prefix) + const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/` + const names = new Set() + for (const p of paths) { + const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null + if (rest === null) continue + const slash = rest.search(/[/\\]/) + if (slash > 0) names.add(rest.slice(0, slash)) + } + return [...names] + } + /** * Remove every object under a storage-root-relative prefix. The filesystem * adapter overrides this with a recursive directory removal; this default From 417ddb5143c7aa4bd33068cdc1a0050a4394ebc7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:13:24 -0700 Subject: [PATCH 23/42] perf(open): answer "are there any entities?" with one directory read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 7.x-to-8.0 layout probe runs on the open path of every store that does not yet carry its completion marker — a restore, a store built by an older release — and asked whether the canonical tree holds anything by LISTING it: a recursive walk of every file in every entity directory, to learn a boolean. It now asks the one-level door added for generation discovery, falling back to the listing on an adapter that lacks it. Also files a defect found while ratifying the operator set: the VFS builds its path-prefix filter as `$startsWith`, an operator no engine spelling accepts, so vfs.searchFiles({ path }) throws INVALID_QUERY on every call that passes a path. Pre-existing, unrelated to the operator work, and left as a filing — a path-prefix search needs a design answer, not a spelling correction. --- src/brainy.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 711d8d02..c479fcc3 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -16732,8 +16732,19 @@ export class Brainy implements BrainyInterface { if (legacyEntityPaths.length === 0) { // Already flat (root entities, no head-branch entities) → stamp the marker // so future opens short-circuit. A genuinely empty/fresh dir gets no marker. - const rootEntities = await probe.listRawObjects('entities') - if (rootEntities.length > 0) { + // "Are there any entities?" is answered by ONE directory read, not by a + // recursive listing of every file in the tree: this runs on the open path + // of every store that does not yet carry the marker (a restore, a store + // built by an older release), and on a large store that listing walks the + // whole canonical tree to learn a boolean. + const oneLevel = ( + probe as unknown as { listRawPrefixes?: (prefix: string) => Promise } + ).listRawPrefixes + const hasRootEntities = + typeof oneLevel === 'function' + ? (await oneLevel.call(probe, 'entities')).length > 0 + : (await probe.listRawObjects('entities')).length > 0 + if (hasRootEntities) { await probe.writeRawObject('_system/migration-layout.json', { layout: 'flat-v8', version: 8, From fb1da1c56dbe8acca7d8cfb6fb66e4446205eaac Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:25:47 -0700 Subject: [PATCH 24/42] perf(idle): the flush-request watch is event-driven; the heartbeat is observability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three idle-burn items from the steady-state audit, and one correction. THE FLUSH-REQUEST WATCH (the strongest of them). It readdir'd the request directory every 500 ms, per brain, for the life of every writer — armed on every non-reader brain whether or not any inspector process existed. In a process holding many stores that is tens of directory reads per second on a completely idle service, plus a stale-request GC on every one of them. It now uses fs.watch, so the arrival itself wakes it and a request is seen SOONER than the poll saw it. Two concessions ride along, both stated in the code: a 30s safety sweep (fs.watch drops events on some network and fuse filesystems, and the GC needs a tick of its own — two orders of magnitude fewer reads than the poll made), and a fall back to the original 500 ms poll, narrated, on a filesystem that cannot watch at all, because an inspector whose request is never seen waits forever. THE WRITER HEARTBEAT goes 10s → 60s. It is observability ONLY — staleness is decided by pid liveness and the fence compares pid + hostname, so no decision anywhere reads the timestamp — and at 10s it was a lock-file write every ten seconds per brain forever, for a value nothing computes with. An operator still sees a heartbeat inside the minute. THE HEALTH NARRATION dedupes by CONTENT, not by the provider's generation counter. That counter bumps on every ledger mutation and rebuild boundary, so a provider bumping it on routine work re-emitted the same unchanged line on every read, while one that never bumped could suppress a line whose reasons had genuinely changed. The generation is still reported; it no longer decides whether the line is worth saying. CORRECTION, and it is against my own earlier claim: the idle-flush commit read a reported idle-CPU observation (many stores, no writes, a flush every ~35s, over a core burned) as caused by the flush path. That does not follow — this engine's cadence is write-driven (every trigger runs through noteWriteForPersistence, which only a committed write calls), so something was CALLING flush() on those brains and the caller is still unidentified. The clean-flush gate makes such a call free; it does not account for it. The code comments and the idle lane now say exactly that. Pins: tests/integration/flush-watcher-event-driven.test.ts — an idle writer makes at most one request-directory read in 8 seconds (the old poll made ~16), and a dropped request is still acked well inside the safety sweep. --- src/brainy.ts | 54 +++++--- src/storage/adapters/fileSystemStorage.ts | 116 +++++++++++++++--- .../flush-watcher-event-driven.test.ts | 94 ++++++++++++++ tests/integration/idle-costs-nothing.test.ts | 17 ++- 4 files changed, 245 insertions(+), 36 deletions(-) create mode 100644 tests/integration/flush-watcher-event-driven.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index c479fcc3..3426dcce 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -749,12 +749,18 @@ export class Brainy implements BrainyInterface { * Whether a write has been committed since the last flush that ran. THE * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written * to has nothing to make durable, and a flush over it must cost nothing and - * say nothing. Measured on a production process holding 21 brains: with no - * writes for ten minutes it still printed "All indexes flushed to disk in - * 216–601ms" per brain every ~35s and idled at 1.26 cores, because a flush - * called every provider, stamped the watermarks, persisted the generation - * counter and re-stamped the entity tree whether or not anything had - * changed. + * say nothing. Before this, a flush called every provider, stamped the + * watermarks, persisted the generation counter and re-stamped the entity + * tree whether or not anything had changed — roughly 28 writes for a store + * that had not moved. + * + * WHAT THIS DOES NOT EXPLAIN, stated so nobody reads it as solved: a + * production process holding 21 brains printed "All indexes flushed to disk + * in 216-601ms" per brain every ~35s and idled at 1.26 cores with no writes + * for ten minutes. This engine's cadence is WRITE-DRIVEN — every trigger + * runs through noteWriteForPersistence, which only a committed write calls — + * so something was calling flush() on those brains, and this gate makes such + * a call free rather than accounting for it. The caller is still unidentified. */ private _dirtySinceLastFlush = false private _persistIdleTimer: ReturnType | null = null @@ -851,7 +857,18 @@ export class Brainy implements BrainyInterface { // Read-gate narration dedup: a degraded-but-serving or not-ready health // report narrates via prodLog.warn ONCE per (provider, report.generation) — // never once per read. Keyed on the provider instance itself. - private _lastNarratedHealthGeneration = new Map() + /** + * The last health narration emitted per provider, keyed by its CONTENT. + * + * This used to dedupe on the provider's `generation` counter, which bumps on + * every ledger mutation and every rebuild boundary — so a provider that + * bumps its generation on routine work re-emitted the same unchanged health + * line on every read that consulted it, and a provider that never bumped + * could suppress a line whose reasons had genuinely changed. The dedupe key + * is now what the line SAYS: an unchanged verdict is silent however the + * generation moves, and a changed verdict is always heard. + */ + private _lastNarratedHealth = new Map() constructor(config?: BrainyConfig) { // The reserved-field write policy died with the field-addressing law: @@ -12366,11 +12383,11 @@ export class Brainy implements BrainyInterface { // committed since the last flush, so every step below would re-persist // state identical to what is already on disk — provider flushes, the // watermark stamps, the generation counter, the entity-tree stamp — and - // print two lines announcing it. On a process holding 21 brains that - // no-op cost 1.26 cores at idle. The witness is set by every committed + // print two lines announcing it. The witness is set by every committed // write (see noteWriteForPersistence) and cleared here; a write landing // DURING this flush sets it again, so it is never lost — the next flush - // does that write's work. + // does that write's work. This makes an unexplained flush FREE; it does + // not explain one (see _dirtySinceLastFlush). if (!this._dirtySinceLastFlush) { return } @@ -17243,12 +17260,17 @@ export class Brainy implements BrainyInterface { if (assessment.reasons.length > 0 && assessment.report != null) { const generation = assessment.report.generation - if (this._lastNarratedHealthGeneration.get(provider) !== generation) { - this._lastNarratedHealthGeneration.set(provider, generation) - prodLog.warn( - `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + - assessment.reasons.join('; ') - ) + // Dedupe by CONTENT, not by the provider's generation counter — see + // _lastNarratedHealth. The generation is still REPORTED (an operator + // wants to know which generation produced the verdict); it just no + // longer decides whether the line is worth saying. + const line = + `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + + assessment.reasons.join('; ') + const key = `${assessment.report.provider}\u0000${assessment.reasons.join('; ')}` + if (this._lastNarratedHealth.get(provider) !== key) { + this._lastNarratedHealth.set(provider, key) + prodLog.warn(line) } } diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 3f1055c2..9d04b46d 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -107,7 +107,23 @@ export class FileSystemStorage extends BaseStorage { * "the previous writer died" without inferring either from a pid. */ private static readonly WRITER_CLOSE_FILE = '_writer.close' - private static readonly WRITER_HEARTBEAT_MS = 10_000 + /** + * How often the lock file's `lastHeartbeat` is rewritten. + * + * THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness + * is decided by PID LIVENESS alone (see isWriterLockStale) and the fence + * compares pid + hostname — no decision anywhere reads this timestamp. It + * exists so an operator inspecting a lock file, or reading the + * BRAINY_WRITER_LOCKED error, can judge liveness themselves. + * + * At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1 + * writes/s across a production process holding 21 idle brains, for a + * human-readable timestamp nothing computes with. At 60s an operator still + * sees a heartbeat inside the minute, at a sixth of the cost. With the + * clean-close record now recording orderly releases explicitly, the + * heartbeat carries even less weight than it did. + */ + private static readonly WRITER_HEARTBEAT_MS = 60_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout private writerLockInfo?: WriterLockInfo @@ -135,9 +151,16 @@ export class FileSystemStorage extends BaseStorage { private static readonly FLUSH_REQUEST_DIR = '_flush_requests' private static readonly FLUSH_RESPONSE_DIR = '_flush_responses' private static readonly FLUSH_WATCH_INTERVAL_MS = 500 + /** + * The safety sweep behind the fs.watch: catches events an exotic filesystem + * dropped, and runs the stale-request GC. See startFlushRequestWatcher. + */ + private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000 private static readonly FLUSH_POLL_INTERVAL_MS = 100 private static readonly FLUSH_REQUEST_TTL_MS = 60_000 private flushWatcherInterval?: NodeJS.Timeout + /** The inotify-backed watch on the request directory, when the FS supports one. */ + private flushWatcher?: import('node:fs').FSWatcher private flushWatcherInFlight = false private flushWatcherOnRequest?: () => Promise @@ -2385,36 +2408,101 @@ export class FileSystemStorage extends BaseStorage { /** * Start watching for cross-process flush requests. Called by Brainy.init() - * in writer mode. Polls `locks/_flush_requests/` every - * FLUSH_WATCH_INTERVAL_MS — each new `.req` file triggers the supplied - * callback (`brain.flush()`), after which an `.ack` is written to - * `locks/_flush_responses/` with the same request ID. Stale `.req` files - * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick. + * in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers + * the supplied callback (`brain.flush()`), after which an `.ack` is written + * to `locks/_flush_responses/` with the same request ID. Stale `.req` files + * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep. + * + * THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request + * directory every 500 ms, per brain, for the entire life of every writer — + * armed on every non-reader brain whether or not any inspector process + * existed. MEASURED on a production process holding 21 brains: 42 directory + * reads per second on a completely idle service, plus a stale-request GC + * pass on every one of them. The engine does no periodic work without a + * cause, and a request that has not been made is not a cause. + * + * `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is + * seen SOONER than the old poll saw it. Two honest concessions ride with it: + * - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because + * `fs.watch` can miss events on network and fuse filesystems and because + * the stale-request GC needs some tick of its own. At 30s that is 0.7 + * reads/s across 21 brains where the poll cost 42. + * - a filesystem that cannot watch at all falls back to the ORIGINAL + * 500 ms poll, narrated once, because correctness outranks idle cost: + * an inspector whose request is never seen waits forever. */ public override startFlushRequestWatcher(onRequest: () => Promise): void { - if (this.flushWatcherInterval) return // already watching + if (this.flushWatcherInterval || this.flushWatcher) return // already watching this.flushWatcherOnRequest = onRequest const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR) - // Ensure both dirs exist up front so the first .req drop doesn't race with mkdir. - this.ensureDirectoryExists(reqDir).catch(() => {}) - this.ensureDirectoryExists(ackDir).catch(() => {}) - - this.flushWatcherInterval = setInterval(() => { - if (this.flushWatcherInFlight) return // skip overlapping tick + const sweep = (): void => { + if (this.flushWatcherInFlight) return // skip overlapping sweep this.flushWatcherInFlight = true this.processFlushRequests(reqDir, ackDir).finally(() => { this.flushWatcherInFlight = false }) - }, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) + } + + // Ensure both dirs exist up front so the first .req drop doesn't race with + // mkdir — and so there is a directory to watch. + void this.ensureDirectoryExists(reqDir) + .then(() => this.ensureDirectoryExists(ackDir)) + .then(() => { + if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile + try { + const watcher = fs.watch(reqDir, () => sweep()) + this.flushWatcher = watcher + watcher.on('error', (err: Error) => { + // A watch that dies mid-life must not leave the door deaf. + console.warn( + `[brainy] Flush-request watch failed (${err.message}) — falling back to polling.` + ) + this.flushWatcher?.close() + this.flushWatcher = undefined + this.startFlushRequestPolling(sweep) + }) + if (typeof watcher.unref === 'function') watcher.unref() + // The safety sweep: missed events on exotic filesystems, and the + // stale-request GC. + this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS) + if (typeof this.flushWatcherInterval.unref === 'function') { + this.flushWatcherInterval.unref() + } + // One sweep now: a request may have been dropped before the watch armed. + sweep() + } catch (err) { + console.warn( + `[brainy] Flush-request directory cannot be watched on this filesystem ` + + `(${(err as Error).message}) — polling every ` + + `${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.` + ) + this.startFlushRequestPolling(sweep) + } + }) + .catch(() => { + // The request directory could not be created; nothing to watch. A + // cross-process flush request cannot be made either, so there is + // nothing to miss. + }) + } + + /** The original 500 ms poll — the fallback when a directory cannot be watched. */ + private startFlushRequestPolling(sweep: () => void): void { + if (this.flushWatcherInterval) return + this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) if (typeof this.flushWatcherInterval.unref === 'function') { this.flushWatcherInterval.unref() } } public override stopFlushRequestWatcher(): void { + if (this.flushWatcher) { + this.flushWatcher.close() + this.flushWatcher = undefined + } if (this.flushWatcherInterval) { clearInterval(this.flushWatcherInterval) this.flushWatcherInterval = undefined diff --git a/tests/integration/flush-watcher-event-driven.test.ts b/tests/integration/flush-watcher-event-driven.test.ts new file mode 100644 index 00000000..4b2e80c4 --- /dev/null +++ b/tests/integration/flush-watcher-event-driven.test.ts @@ -0,0 +1,94 @@ +/** + * @module tests/integration/flush-watcher-event-driven + * @description THE FLUSH-REQUEST WATCH IS EVENT-DRIVEN. + * + * It used to `readdir` the request directory every 500 ms, per brain, for the + * life of every writer — armed on every non-reader brain whether or not any + * inspector process existed. MEASURED on a production process holding 21 + * brains: 42 directory reads per second on a completely idle service, plus a + * stale-request GC pass on every one of them. + * + * The law: a request that has not been made is not a cause. The arrival itself + * wakes the watcher, so the request is seen SOONER than the poll saw it, and a + * slow safety sweep covers filesystems that drop watch events and the GC. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' +import * as nodeFs from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +describe('the flush-request watcher', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function openWriter(): Promise<{ brain: Brainy; dir: string }> { + const dir = mkdtempSync(join(tmpdir(), 'brainy-flush-watch-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + await brain.add({ data: 'a row', type: NounType.Concept }) + await brain.flush() + return { brain, dir } + } + + it('does not poll the request directory on an idle writer', async () => { + const { dir } = await openWriter() + const reqDir = join(dir, 'locks', '_flush_requests') + + // Count real reads of the request directory over a window far longer than + // the old 500ms poll (which would have made ~16 of them). + const realReaddir = nodeFs.promises.readdir + let requestDirReads = 0 + const spy = vi + .spyOn(nodeFs.promises, 'readdir') + .mockImplementation((async (p: unknown, ...rest: unknown[]) => { + if (String(p) === reqDir) requestDirReads++ + return (realReaddir as unknown as (...a: unknown[]) => Promise)(p, ...rest) + }) as typeof nodeFs.promises.readdir) + + await new Promise((r) => setTimeout(r, 8_000)) + spy.mockRestore() + + // The old poll: 500ms → ~16 reads. The safety sweep is 30s → 0 in this window. + expect(requestDirReads).toBeLessThanOrEqual(1) + }, 120_000) + + it('answers a request that arrives, without waiting for the sweep', async () => { + const { brain, dir } = await openWriter() + const reqDir = join(dir, 'locks', '_flush_requests') + const ackDir = join(dir, 'locks', '_flush_responses') + mkdirSync(reqDir, { recursive: true }) + + // Drop a request exactly as an out-of-process inspector does. + const id = 'test-request-0001' + writeFileSync(join(reqDir, `${id}.req`), JSON.stringify({ at: Date.now() })) + + // The ack must land far sooner than the 30s safety sweep. + const deadline = Date.now() + 10_000 + let acked = false + while (Date.now() < deadline) { + try { + const entries = await nodeFs.promises.readdir(ackDir) + if (entries.some((e) => e.startsWith(id))) { acked = true; break } + } catch { /* dir not created yet */ } + await new Promise((r) => setTimeout(r, 100)) + } + expect(acked, 'the watcher must answer an arriving request').toBe(true) + void brain + }, 120_000) +}) diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts index 8f951d46..b5c386cf 100644 --- a/tests/integration/idle-costs-nothing.test.ts +++ b/tests/integration/idle-costs-nothing.test.ts @@ -2,12 +2,17 @@ * @module tests/integration/idle-costs-nothing * @description AN IDLE BRAIN DOES NO WORK. * - * Measured on a production process holding 21 brains: with no writes for ten - * minutes it printed "All indexes flushed to disk in 216–601ms" per brain - * every ~35 seconds and idled at 1.26 cores. Every one of those flushes - * re-persisted state identical to what was already on disk — the provider - * flushes, the watermark stamps, the generation counter, the entity-tree - * stamp — because `flush()` never asked whether anything had changed. + * A flush used to re-persist state identical to what was already on disk — + * the provider flushes, the watermark stamps, the generation counter, the + * entity-tree stamp, roughly 28 writes — because `flush()` never asked whether + * anything had changed. + * + * The field observation that started this: a production process holding 21 + * brains printed "All indexes flushed to disk in 216–601ms" per brain every + * ~35 seconds and idled at 1.26 cores, with no writes for ten minutes. This + * engine's cadence is WRITE-DRIVEN, so that observation is NOT explained by + * the cadence and is not claimed to be fixed here — what is fixed is that such + * a call now costs nothing. Who was calling flush() remains open. * * The laws pinned here: * (a) the persistence cadence arms only on a write — a brain nobody writes From 16d2e1a97ec9ba435a682b91c84b88d4e87bfd95 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:30:00 -0700 Subject: [PATCH 25/42] fix(storage): the flush watcher cannot arm twice in its async window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Arming is asynchronous — the request directory is created before it can be watched — so during that window neither the watcher nor the sweep interval exists yet and the guard let a second call through, leaving two watchers and two sweeps for the life of the store. The callback is the flag that covers the window. --- src/storage/adapters/fileSystemStorage.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 9d04b46d..fa0715df 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -2432,7 +2432,12 @@ export class FileSystemStorage extends BaseStorage { * an inspector whose request is never seen waits forever. */ public override startFlushRequestWatcher(onRequest: () => Promise): void { - if (this.flushWatcherInterval || this.flushWatcher) return // already watching + // Already watching — or already ARMING. The arm is asynchronous (the + // request directory is created before it can be watched), so neither the + // watcher nor the interval exists yet during that window; the callback is + // the flag that covers it. Without this a second call in the window would + // leave two watchers and two sweeps running for the life of the store. + if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return this.flushWatcherOnRequest = onRequest const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) From 5c22f9500ce9628e7652613babefb00e094adedd Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:31:57 -0700 Subject: [PATCH 26/42] fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The safety sweep is armed alongside the watch, and startFlushRequestPolling() declines to arm over an existing interval — so when a watch died mid-life the fallback did nothing and the store quietly answered flush requests on a 30s cadence instead of the 500ms one the door promises. The sweep is cleared first. A degrade nobody asked for is still a degrade. --- src/storage/adapters/fileSystemStorage.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index fa0715df..5ec1d88e 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -2467,6 +2467,15 @@ export class FileSystemStorage extends BaseStorage { ) this.flushWatcher?.close() this.flushWatcher = undefined + // The SAFETY sweep must go first. It is already armed at 30s, and + // startFlushRequestPolling() declines to arm over an existing + // interval — so leaving it would quietly leave this store answering + // flush requests on a 30s cadence instead of the 500ms one the door + // promises. A degrade nobody asked for is still a degrade. + if (this.flushWatcherInterval) { + clearInterval(this.flushWatcherInterval) + this.flushWatcherInterval = undefined + } this.startFlushRequestPolling(sweep) }) if (typeof watcher.unref === 'function') watcher.unref() From 2cf3801007a18a0add42b0b6cfe75c245bde556d Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:56:26 -0700 Subject: [PATCH 27/42] feat(open): name the two steps that hold the vfs-bootstrap phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a 14,056-noun / 72,679-verb production-shaped store, measured solo under an exclusive lock: the vfs-bootstrap phase costs 37.8s on main and 38.0s on this branch — unchanged — and NO "vfs.init" step line was emitted at all, meaning the VFS's own init fell under the 2s narration threshold. The phase is therefore almost entirely NOT the VFS, and the old-root sweep this branch moved to the background was never what made it expensive. What else lives in that span is now named: the log-authority artifact read, the adoption ORACLE (which verifies the log against canonical before flipping a brain to durable-at-ack), the legacy pending-embed sidecar bridge, and the pending-embed recovery fold. One of those holds ~38 seconds of every open of this store and the next measurement will say which, by name, instead of leaving a phase label to be guessed at. --- src/brainy.ts | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 3426dcce..70d46973 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1760,7 +1760,11 @@ export class Brainy implements BrainyInterface { const storedArtifact = await this.storage .readRawObject(LOG_AUTHORITY_PATH) .catch(() => null) - const authority = await readLogAuthority(this.storage) + const authority = await step( + 'read-log-authority', + 'reading the stored storage-authority artifact', + () => readLogAuthority(this.storage) + ) this._logAuthority = authority if (authority.authority === 'log') { this.generationStore.setLogDurability('at-ack') @@ -1771,7 +1775,12 @@ export class Brainy implements BrainyInterface { this.generationStore.getFactLog() !== null ) { try { - await this.adoptLogAuthority() + await step( + 'adopt-log-authority', + 'the adoption oracle: verifying the log against canonical before flipping this ' + + 'brain to durable-at-ack, and backfilling any curable divergence', + () => this.adoptLogAuthority() + ) prodLog.info( '[Brainy] storage authority adopted at open: generation log ' + '(fleet default; oracle green; durable-at-ack enabled)' @@ -1811,8 +1820,16 @@ export class Brainy implements BrainyInterface { // this is where it lands. if (!this.isReadOnly) { try { - await this.bridgeLegacyPendingEmbedSidecars() - await this.recoverPendingEmbedsFromLog() + await step( + 'bridge-pending-embed-sidecars', + 'migrating any pre-log deferred-embed marker files into the generation log', + () => this.bridgeLegacyPendingEmbedSidecars() + ) + await step( + 'recover-pending-embeds', + 'folding the generation log\'s deferred-embed markers back into the pending set', + () => this.recoverPendingEmbedsFromLog() + ) if (this._pendingEmbedIds.size > 0) { prodLog.info( `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + From 02c61636370387f6c3ebc38be1087f58e0a14033 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:04:49 -0700 Subject: [PATCH 28/42] docs: measurements in public history carry numbers, not provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A release audit found hostnames, store identities and operational anecdotes in this branch's commit messages — not trade secrets, but nothing a public repository's permanent history should carry either. The messages were rewritten to keep every number and drop every provenance; the rule is written down here so the next measurement does not have to be caught by an audit. --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 50860cb5..84bd25e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,6 +57,13 @@ see `package.json` for `test:integration`, `test:coverage`, and friends. description states a number, cite the benchmark that produced it (see [docs/performance-envelopes.md](docs/performance-envelopes.md) for the pattern). Don't state an estimate as if it were measured. +- **Measurements carry numbers, not provenance.** Public commit messages and + docs give the SHAPE a number was taken at and never where it was taken: no + hostnames, no store or deployment identities, no operational anecdotes about + someone's running system. "A 14,056-noun / 72,679-verb production-shaped + store, measured solo under an exclusive lock" tells a reader everything the + number depends on; the machine it ran on and whose data it was tell them + nothing except where somebody's infrastructure lives. ## License From 61a469270e476e3ae64755069a39794303bfafd9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:05:46 -0700 Subject: [PATCH 29/42] =?UTF-8?q?docs(releases):=2010.4.4=20consumer=20not?= =?UTF-8?q?es=20=E2=80=94=20correctness=20and=20observability,=20with=20th?= =?UTF-8?q?e=20performance=20line=20stated=20exactly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 4d716efa..e8833b80 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,115 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.4.4 — 2026-08-28 + +**A correctness and observability release.** The headline is not speed: it is that a +restart now tells you the truth about itself, a store stops lying about how much it +holds, and the engine stops doing work nobody asked for. There is a performance +improvement and it is modest; it is stated exactly below rather than rounded up. + +### The dark restart — fixed at the root + +A service could stop cleanly, exit 0, having awaited `close()` on every store it held, +and its next boot would announce `Overwriting stale writer lock … appears dead` for +every one of them. Nothing had crashed. Two deployments hit this; the same defect also +made those boots pay a crash-recovery fold they did not owe. + +The cause was not the lock. `close()` released it correctly — when it got there. A +failure part-way through close skipped both the release AND the clean-shutdown marker, +and "the recorded pid is gone" reads identically for an orderly restart and a crash. + +- `close()` is now two parts and the second is unconditional: the flush-request watcher, + the **writer lock**, the VFS timers and the terminal `closed` flag are released whether + the durable steps succeeded or not. The original failure is narrated with what it costs + the next open, then rethrown. +- Releasing the lock writes a **clean-close record** naming the lock generation it gave + up. The next open reads that record instead of guessing: recorded → nothing to recover; + absent → it says so, and names the recovery it is about to run. This also ends two + long-standing false alarms — a recycled pid locking a store out of its own reopen, and + `Re-acquiring writer lock … this is a bug` after a perfectly clean close. +- The signal path stopped failing in a batch. One store's failing flush used to strand + every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the + generation store's close (the marker) is part of shutdown, the lock goes in a `finally`, + and the handler no longer calls `process.exit()` when the host application has its own + signal handler, a race that truncated the host's own shutdown mid-flight. + +### The count ledger stops lying, and `counts.json` is written atomically + +The all-tier scalars are the denominator a coverage check subtracts against. A ledger +derived under the old rule — one entity per id DIRECTORY — counted ghost and scar +containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for +the life of the store. Two copies of one archive could disagree, and a downstream index +heal reported remaining work that did not exist. + +- Such a ledger now derives itself honestly **in the background** after the open, counting + identity records, and persists the correction stamped. Nothing waits for it, because no + read is served from a denominator. +- A derivation that raced a write refuses to stamp its number: one retry on a quiet store, + then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under + a barrier. +- `counts.json` is written temp+rename. A truncating write left a window in which a + concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open + down the full-rescan path, so the cheapest file in the store was buying the most + expensive recovery. + +### An open and a repair narrate themselves — on a channel a log level cannot silence + +A store could open for three minutes and print nothing at all. The phase timings existed; +they were written to a channel that every production-looking environment clamps away. + +- Narration moved to an always-visible channel. An open now heartbeats the phase it is in, + names each phase as it ends with what it was paying for, and names the expensive STEP + inside a phase. `repairIndex()` does the same and its receipt carries a per-family + `durationMs` — a repair that ran for half an hour with no output could only be watched + through `top`. +- A brain nobody has written to now does nothing: a flush over a clean store is a no-op + and says nothing, the graph index's auto-flush asks before it acts, and the + cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a + directory every 500 ms per store forever, with a slow safety sweep behind it and a + narrated fall back to polling where a filesystem cannot be watched. +- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()` + does not wait for it, every other family serves, and that family's doors refuse **by + name, carrying the provider's own progress**, saying plainly that they open by + themselves and no action is needed. Health narration dedupes by content, so an unchanged + verdict is silent however a provider's generation counter moves. + +### For operators — one behaviour change + +**Four `where` operators that previously returned an empty page now raise +`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range +posting index cannot evaluate a substring, a pattern or an array length without reading +every row, and it now refuses by name instead of answering with an empty result that +looks like an answer. + +**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and +`excludes`. All 25 accepted operator tokens now agree between this engine and its +accelerated counterpart. + +### Performance — stated exactly + +Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under +an exclusive lock: + +- **Warm reopen after a clean close: 85.7 s → 77.0 s (−10.2%).** The whole of that gain is + one fix — generation discovery reads directory NAMES instead of recursively walking the + entire generation log (−9.2 s, and it scales with history rather than row count). The + VFS phase is **unchanged**. +- **Cold open: −31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving + off the critical path accounts for storage-init dropping 5,941 ms → 25 ms. +- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own + init is under 2 s of that phase. It is the log-authority adoption and/or the + pending-embed log recovery, both now instrumented so the next measurement names the + culprit outright. + +Continuing work, named so nobody has to rediscover it: that ~38 s term; making the +generation store's committed-range set lazy; the hydration path that substitutes +`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix +filter built with a `$startsWith` spelling no operator set accepts, so +`searchFiles({ path })` throws today. + +--- + ## v10.4.3 — 2026-08-27 (Open Brainy's first release) **`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for From a8c724a202a9bad3ac6d5da20e08209478af2f2d Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:08:19 -0700 Subject: [PATCH 30/42] docs: the contract manifest stands alone; public docs describe this engine only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest's prose pointer named a document that answers a confidential specification, and such a document does not belong in a public repository even in summary. The pointer is dropped — the manifest is generated from this engine's own surface and is self-describing — and the requirement marking it deliberately omits is recorded with the contract's owner rather than here. The standard is written down so this is not relitigated per document. --- CONTRIBUTING.md | 4 ++++ docs/api-contract.json | 1 - scripts/emit-contract-manifest.mjs | 5 ++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84bd25e3..54d4f784 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,10 @@ see `package.json` for `test:integration`, `test:coverage`, and friends. store, measured solo under an exclusive lock" tells a reader everything the number depends on; the machine it ran on and whose data it was tell them nothing except where somebody's infrastructure lives. +- **Documents that answer or reference a confidential specification never enter + this repository, even summarized.** The public docs describe THIS engine and + the published contract, and nothing else — a summary of a private document is + still that document's contents. ## License diff --git a/docs/api-contract.json b/docs/api-contract.json index 9dadcc0e..aafd838a 100644 --- a/docs/api-contract.json +++ b/docs/api-contract.json @@ -1,7 +1,6 @@ { "contractVersion": 1, "engine": "@soulcraftlabs/brainy", - "prose": "docs/contract-1-ratification.md", "compatibility": { "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs index 13a9bc6d..be73d4ca 100644 --- a/scripts/emit-contract-manifest.mjs +++ b/scripts/emit-contract-manifest.mjs @@ -10,8 +10,8 @@ * between two engines, never between two authors. * * Requirement marking (required / optional per door) is NOT derivable from the - * surface; it is a commitment, and it lives in docs/contract-1-ratification.md. - * This manifest carries the surface; that document carries the promise. + * surface — it is a commitment, recorded with the contract's owner rather than + * here. This manifest carries the surface; the promise lives with the contract. * * Usage: node scripts/emit-contract-manifest.mjs [--check] * --check exits non-zero when the committed manifest is stale. @@ -68,7 +68,6 @@ const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) const manifest = { contractVersion: versionModule.contractVersion(), engine: '@soulcraftlabs/brainy', - prose: 'docs/contract-1-ratification.md', compatibility: { minor: 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', From 42e2da259bcdc27b08087b3eda19e16ac77d7d65 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:27:40 -0700 Subject: [PATCH 31/42] fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gate failures on main, one real and one long-hidden. THE HEALTH-GATE PIN encoded the old law — "narrates once per generation, twice across a generation bump" — which the content-keyed dedupe deliberately replaced. A provider's `generation` bumps on every ledger mutation and every rebuild boundary, so keying narration on it re-printed an unchanged health line on every read that consulted a busy provider, and let a provider that never bumped suppress a line whose reasons had genuinely changed. The pin now asserts BOTH directions: an unchanged verdict stays silent however the counter moves, and a changed verdict is always heard. THE VFS HYBRID-SEARCH SUITE configured its store with `options.basePath`, an alias removed at the 8.0 major that configures nothing. The suite was therefore never using its temp directory — it opened the DEFAULT store, shared with every other run on the machine, and accumulated tens of thousands of rows until it failed on that shared store's graph adjacency instead of on anything it tests. It now passes `storage.path`. The suite drops from 6.5s to 0.3s, which is the measure of how much foreign data it had been opening. Neither failure was caused by the release branch; the first is the branch's own behaviour change meeting its outdated pin, the second predates it. --- tests/integration/health-gate.test.ts | 20 ++++++++++++++++---- tests/integration/hybrid-search-vfs.test.ts | 8 +++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/integration/health-gate.test.ts b/tests/integration/health-gate.test.ts index 1c9a642d..f2952116 100644 --- a/tests/integration/health-gate.test.ts +++ b/tests/integration/health-gate.test.ts @@ -188,7 +188,7 @@ describe('health gate (b) — unledgered is unknown: never blocks a serving prov describe('health gate (c) — degraded-but-serving narrates once per generation', () => { // PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the // degraded report lives on the family the read actually consults. - it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => { + it('a heal:"repair" failure serves; narrates once per DISTINCT VERDICT, not once per generation bump', async () => { const brain = new Brainy(createTestConfig({ silent: true })) await brain.init() brains.push(brain) @@ -197,12 +197,13 @@ describe('health gate (c) — degraded-but-serving narrates once per generation' const internals = internalsOf(brain) let generation = 1 + let detail = 'counter drift' internals.metadataIndex.healthReport = () => healthReport({ provider: 'vector', serving: true, healthy: false, - invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail: 'counter drift' })], + invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail })], generation }) @@ -212,11 +213,22 @@ describe('health gate (c) — degraded-but-serving narrates once per generation' await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(1) // same generation both times — one narration + expect(countNarrations()).toBe(1) // same verdict both times — one narration + // THE DEDUPE KEY IS THE VERDICT, NOT THE COUNTER. A provider's `generation` + // bumps on every ledger mutation and every rebuild boundary, so keying the + // narration on it re-printed an UNCHANGED health line on every read that + // consulted a busy provider — and, in the other direction, let a provider + // that never bumped suppress a line whose reasons had genuinely changed. + // An unchanged verdict is silent however the counter moves: generation = 2 await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(2) // generation bumped — a second narration + expect(countNarrations()).toBe(1) // generation bumped, verdict identical — still silent + + // ...and a CHANGED verdict is always heard, bump or no bump: + detail = 'counter drift widened to 12 rows' + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + expect(countNarrations()).toBe(2) // the reasons changed — a new narration delete internals.metadataIndex.healthReport }) diff --git a/tests/integration/hybrid-search-vfs.test.ts b/tests/integration/hybrid-search-vfs.test.ts index c881fa97..219c4c83 100644 --- a/tests/integration/hybrid-search-vfs.test.ts +++ b/tests/integration/hybrid-search-vfs.test.ts @@ -21,10 +21,16 @@ describe('Hybrid Search with VFS', () => { testDir = path.join(os.tmpdir(), `brainy-hybrid-vfs-test-${Date.now()}`) fs.mkdirSync(testDir, { recursive: true }) + // `storage.path`, NOT the pre-8.0 `options.basePath` alias. That alias was + // removed at the 8.0 major and configures nothing, so this suite silently + // opened the DEFAULT store instead of its own temp directory — sharing one + // on-disk brain with every other run on the machine, accumulating tens of + // thousands of rows, and eventually failing on that shared store's graph + // adjacency rather than on anything it was written to test. brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', - options: { basePath: testDir } + path: testDir } }) await brain.init() From d49148e1407cc14a5de031f605e081bff30cb399 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:30:30 -0700 Subject: [PATCH 32/42] fix(vfs): the old-root sweep narrates only when it has something to say MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release gate's own output caught this: every test brain printed "[VFS] old-root sweep complete in 1ms and recorded" — hundreds of lines — and a consumer would get two of them on the first open of every store. They were emitted on the always-visible channel, which a production log level deliberately CANNOT silence. That channel exists so an operator can always learn why a database is slow; a 0ms no-op on a fresh store is not that, and announcing it there trains people to ignore the one channel built to be impossible to ignore. It was also inconsistent with every other narration in this work, all of which is silent under a threshold. The sweep now speaks when it has something to say — duplicate roots removed, or a wall over a second that a person watching a slow first open deserves explained — and otherwise does its work, records its marker, and stays quiet. cleanupOldRoots() reports what it removed so the decision rests on a fact rather than on a guess. Pin: a fresh store's sweep emits nothing on the channel and still records its marker, so the silence can never be mistaken for the work being skipped. --- src/vfs/VirtualFileSystem.ts | 40 ++++++++++++++----- tests/integration/vfs-root-sweep-once.test.ts | 27 +++++++++++++ 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 19470b48..46c6a12d 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -76,6 +76,11 @@ export class VirtualFileSystem implements IVirtualFileSystem { * `_system/`, like every other marker there — never enumerated as data. */ private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json' + /** + * Below this wall, a sweep that removed nothing says nothing — see + * {@link sweepOldRootsIfNeeded}. + */ + private static readonly ROOT_SWEEP_NARRATE_MS = 1_000 private currentUser: string = 'system' // Track current user for collaboration // Knowledge Layer features available via augmentation (brain.use('knowledge')) @@ -434,21 +439,31 @@ export class VirtualFileSystem implements IVirtualFileSystem { } catch { // Unreadable marker: sweep, and rewrite it below. } - prodLog.narrate( - '[VFS] one-time sweep for pre-fixed-id root directories running in the background — ' + - 'the open does not wait for it, and once it has run this store never sweeps again.' - ) + // NARRATION HAS A THRESHOLD, like every other line this engine emits on the + // always-visible channel. On a fresh or small store this sweep finds + // nothing and costs a millisecond, and announcing it — twice — on a + // channel a production log level deliberately CANNOT silence would train + // operators to ignore the one channel that exists to be impossible to + // ignore. It speaks when it has something to say: duplicates removed, or a + // wall long enough that somebody watching a slow first open deserves to + // know what is running. Otherwise it does its work and stays quiet. const startedAt = Date.now() - await this.cleanupOldRoots() + const duplicatesRemoved = await this.cleanupOldRoots() + const elapsedMs = Date.now() - startedAt try { await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, { sweptAt: new Date().toISOString(), - durationMs: Date.now() - startedAt + durationMs: elapsedMs }) - prodLog.narrate( - `[VFS] old-root sweep complete in ${Date.now() - startedAt}ms and recorded — ` + - 'no future open pays for it.' - ) + if (duplicatesRemoved > 0 || elapsedMs >= VirtualFileSystem.ROOT_SWEEP_NARRATE_MS) { + prodLog.narrate( + `[VFS] one-time old-root sweep complete in ${elapsedMs}ms` + + (duplicatesRemoved > 0 + ? `, ${duplicatesRemoved} pre-fixed-id root(s) removed` + : '') + + ' and recorded — no future open pays for it.' + ) + } } catch (error) { // Unrecorded sweep = the next open sweeps again. Conservative, and said // out loud rather than quietly repeated forever. @@ -495,7 +510,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { return null } - private async cleanupOldRoots(): Promise { + private async cleanupOldRoots(): Promise { + let removed = 0 try { // Find any old VFS roots with UUID-based IDs (not our fixed ID) const oldRoots = await this.brain.find({ @@ -517,6 +533,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { for (const duplicate of duplicates) { try { await this.brain.remove(duplicate.id) + removed++ console.log(`VFS: Deleted old root ${duplicate.id.substring(0, 8)}`) } catch (error) { console.warn(`VFS: Failed to delete old root ${duplicate.id}:`, error) @@ -529,6 +546,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Non-critical error - log and continue console.warn('VFS: Cleanup of old roots failed (non-critical):', error) } + return removed } /** diff --git a/tests/integration/vfs-root-sweep-once.test.ts b/tests/integration/vfs-root-sweep-once.test.ts index f84f4413..cac59b70 100644 --- a/tests/integration/vfs-root-sweep-once.test.ts +++ b/tests/integration/vfs-root-sweep-once.test.ts @@ -20,6 +20,7 @@ import { join } from 'node:path' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' +import { prodLog } from '../../src/utils/logger.js' describe('the VFS old-root sweep', () => { const dirs: string[] = [] @@ -80,6 +81,32 @@ describe('the VFS old-root sweep', () => { expect(sweepSpy).not.toHaveBeenCalled() }, 180_000) + it('a sweep that removes nothing on a fresh store says nothing', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-quiet-')) + dirs.push(dir) + + // The always-visible channel cannot be silenced by a log level, so a line + // on it has to earn its place. A fresh store's sweep finds no duplicate + // roots and costs a millisecond — it must do its work, record its marker, + // and stay quiet, or it trains operators to ignore the one channel that + // exists to be impossible to ignore. + const narrated: string[] = [] + const spy = vi.spyOn(prodLog, 'narrate').mockImplementation(((...args: unknown[]) => { + narrated.push(args.map((a) => String(a)).join(' ')) + }) as typeof prodLog.narrate) + + const brain = await open(dir) + await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + spy.mockRestore() + + expect(narrated.filter((l) => /old-root sweep/i.test(l))).toEqual([]) + // ...and it still did the work: the marker is recorded, so no future open sweeps. + expect( + existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || + existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) + ).toBe(true) + }, 180_000) + it('the open does not wait for the sweep', async () => { const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-async-')) dirs.push(dir) From ff39941b0a8795d0dd9fd99a19395f6a9708cebb Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:39:55 -0700 Subject: [PATCH 33/42] chore(release): 10.4.4 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7790837..a54d609e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,35 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) + +- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) +- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25) +- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40) +- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2) +- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927) +- docs: measurements in public history carry numbers, not provenance (02c61636) +- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010) +- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950) +- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9) +- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5) +- perf(open): answer "are there any entities?" with one directory read (417ddb51) +- perf(generations): discover generations by directory name, not by walking the log (9dd39921) +- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc) +- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca) +- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f) +- chore: keep the generated neural stamps at main's values (c1f09723) +- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3) +- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02) +- feat(open): open never waits for a provider that is rebuilding itself (131daa08) +- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f) +- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6) +- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b) +- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f) +- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c) +- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b) + + ### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27) - Merge branch 'next/open-brainy-rename' (a58372f0) diff --git a/package-lock.json b/package-lock.json index 4d247780..c4f66561 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.3", + "version": "10.4.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.3", + "version": "10.4.4", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 60e901de..06ce0253 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.3", + "version": "10.4.4", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From b8475cc86a9c5dca8c5c34f84f1e314e76369ef7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 13:00:42 -0700 Subject: [PATCH 34/42] =?UTF-8?q?fix(release):=20the=20release=20page=20po?= =?UTF-8?q?sts=20to=20this=20repository=20=E2=80=94=20soulcraftlabs/open-b?= =?UTF-8?q?rainy,=20never=20the=20engine's?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 11 POSTed to repos/soulcraft/brainy while printing the correct URL; dormant only because FORGEJO_RELEASE_TOKEN was unset. Found during the 10.4.4 cut verification. --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 08293e3a..67ad1053 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -237,7 +237,7 @@ fi # and RELEASES.md are the record; this just gives The Source's UI a release page). echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then - if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ + if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then echo -e "${GREEN}✅ Release page created on The Source${NC}\n" From 298cb6dacaac9ef80db65a723d65cbd53c15d23e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:07:18 -0700 Subject: [PATCH 35/42] fix(recovery): a torn generation-log tail is a terminal verdict, never a wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of one defect, found by a seeded-SIGKILL crash lane. THE FALSE POSITIVE. stampEntityTree() recorded generationStore.generation() — the ALLOCATED counter, a number a write in flight has claimed and may never commit — while the JSDoc beside it already said the source is the committed generation. Every crash inside a write window therefore produced a spurious verdict at the next open: either 'sourceGeneration N is ahead of the log head N-1' (the allocated generation died with the process) or 'rollup invariant nounCount: stamped X, observed Y' (the recovery fold folded facts the stamp's counts predate). Both told the operator to run repairIndex() — a whole-store recount — for a store that was coherent. Measured before this commit: 4 of 11 SIGKILL cycles on a healthy store raised one of the two. The stamp and the open now both read committedGeneration(), which is what every other open-time watermark in the class already reasons about. THE TERMINAL VERDICT. A stamp still ahead of committed truth after the recovery fold witnesses a generation that is not in the log — the stamp's fsync outlived the tail's, and there is nothing to arrive. That is its own verdict state now ('torn'), never folded in with 'incoherent': the two have opposite cures. A writer open demotes it — the unusable stamped surface is re-derived at the committed generation from the live counters, O(1), straight-line, no loop and no await on external progress, narrated with both count sets, the stamp's path and its committedAt. A read-only open cannot re-stamp, so it says so and names the cure instead of guessing, and still serves. Neither branch waits, and neither locks an owner out of a canonical tree the stamp only describes. Pins: the verifier returns the torn verdict with both generations; a fabricated head-behind-source store narrates precisely, demotes inside a bounded open, serves its rows, and is quiet at the next open (the demotion converges); a read-only open narrates the same verdict and leaves the bytes untouched. --- src/brainy.ts | 121 +++++++++++++++++++- src/db/familyStamp.ts | 32 ++++-- tests/integration/entity-tree-stamp.test.ts | 104 ++++++++++++++++- 3 files changed, 241 insertions(+), 16 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 70d46973..da04577e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -12497,6 +12497,18 @@ export class Brainy implements BrainyInterface { * healed by `repairIndex()`, whose unconditional recount rebuilds the * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write * fault warns loudly but never fails the flush that carried real data. + * + * THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is + * the ALLOCATED counter — a number a write in flight has claimed and may + * never commit. Stamping it made the stamp's generation label a claim about + * counts it was not taken at, and every crash inside a write window then + * produced a spurious verdict at the next open: either `sourceGeneration N + * is ahead of the log head N-1` (the allocated generation died with the + * process) or `rollup invariant 'nounCount': stamped X, observed Y` (the + * recovery fold folded facts the stamp's counts predate). MEASURED on the + * crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on + * a coherent store raised one of those two verdicts, each of them naming + * `repairIndex()` — a whole-store recount — as the cure for nothing. */ private async stampEntityTree(): Promise { if (this.isReadOnly) return @@ -12507,7 +12519,7 @@ export class Brainy implements BrainyInterface { ]) await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { family: 'entity-tree', - sourceGeneration: this.generationStore.generation(), + sourceGeneration: this.generationStore.committedGeneration(), members: { mode: 'rollup', invariants: { nounCount, verbCount } } }) } catch (error) { @@ -12520,16 +12532,24 @@ export class Brainy implements BrainyInterface { /** * @description Open-time coherence check for the entity tree's family stamp: - * compare `sourceGeneration` against the log head and the stamped rollup - * invariants against the live counters. Verdicts: + * compare `sourceGeneration` against the store's COMMITTED generation and + * the stamped rollup invariants against the live counters. Verdicts: * - `coherent` / `absent` (legacy store; first flush stamps) → silent. * - `behind` → benign for the tree (it is written BY the commit; only the * stamp is stale — a crash landed between commit and flush). Refreshed at * the next flush. + * - `torn` → a TORN GENERATION-LOG TAIL, handled by + * {@link demoteTornEntityTreeStamp}: terminal, never a wait. * - `incoherent` → LOUD: the tree or its counters diverged from what was * stamped — `repairIndex()` recounts from canonical and re-stamps. * Never blocks open; a fault reading the stamp is surfaced as unverifiable, * never conflated with absence. + * + * THE COMPARISON IS AGAINST `committedGeneration()`, matching what + * {@link stampEntityTree} writes and what every other open-time watermark in + * this class already reasons about (the fact-scan capability, the metadata / + * graph / HNSW watermark verdicts). Comparing against the allocated counter + * was the one place that disagreed, and disagreeing was the whole defect. */ private async verifyEntityTreeStamp(): Promise { let stamp: FamilyStamp | null @@ -12546,11 +12566,16 @@ export class Brainy implements BrainyInterface { this.storage.getNounCount(), this.storage.getVerbCount() ]) - const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { + const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), { nounCount, verbCount }) - if (verdict.state === 'incoherent') { + if (verdict.state === 'torn') { + await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, { + nounCount, + verbCount + }) + } else if (verdict.state === 'incoherent') { prodLog.warn( `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + `The canonical tree or its counters diverged from the stamped state — run ` + @@ -12564,6 +12589,92 @@ export class Brainy implements BrainyInterface { } } + /** + * @description THE TERMINAL VERDICT for a torn generation-log tail. + * + * A stamp whose `sourceGeneration` sits ABOVE the store's committed + * watermark witnesses a generation that is not in the log: the stamp's fsync + * outlived the tail's. By the time this runs, log-authority recovery has + * already folded every intact fact above the manifest and advanced the + * watermark to cover them — so if the stamp is STILL ahead, the generation + * it names is not merely late, it is GONE. There is nothing to wait for. + * + * That is the whole point of this method. A field report of this class + * (single-process store, abrupt termination mid-fold) described a reopen + * that narrated the tear and then held 100% CPU with zero log growth for + * eight minutes before an operator wiped the directory. A recovery that + * cannot say what it is waiting for has no business spinning; the honest + * answer here is a verdict, taken now, at O(1) cost. + * + * WHAT THE VERDICT DOES — the stamped surface is UNUSABLE, so it is + * discarded rather than believed: the stamped counts describe a generation + * that never became durable, and comparing them against live counters can + * only produce noise. The tree itself is not in question (it IS canonical — + * every commit writes it, and the fold re-applied every after-image the log + * still holds), so the demotion is a re-derivation of this family's verified + * surface at the generation the store can actually show: + * + * - WRITER open → re-stamp at `committedGeneration()` from the live + * counters — exactly what the next flush would write, taken now so the + * tear cannot re-narrate on every subsequent open. Both count sets are + * logged so an operator can see whether anything really moved. + * - READER open → a reader cannot re-stamp. Narrate the same terminal + * verdict with the named cure and carry on serving; a read-only inspector + * is never locked out of a store, and never left waiting either. + * + * BOUNDEDNESS: straight-line code. No loop, no retry, no await on any + * external progress signal — the two counter reads and one stamp write are + * the entire cost, and none of them scales with the store. + */ + private async demoteTornEntityTreeStamp( + stamp: FamilyStamp, + stampSource: number, + head: number, + observed: { nounCount: number; verbCount: number } + ): Promise { + const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {} + const detail = + `[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` + + `generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` + + `generation is ${head} after crash recovery — the stamp's fsync outlived the log tail's, ` + + `and generation ${stampSource} is not in the log to arrive. Stamped rollups ` + + `${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.` + + if (this.isReadOnly) { + prodLog.warn( + `${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` + + `family stays UNVERIFIED for this session (reads are unaffected — the canonical tree ` + + `is the truth this stamp only describes). Cure: open the store with a writer, or run ` + + `brain.repairIndex() there, to recount from canonical and re-stamp.` + ) + return + } + + const startedAt = Date.now() + try { + await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { + family: 'entity-tree', + sourceGeneration: head, + members: { + mode: 'rollup', + invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount } + } + }) + prodLog.warn( + `${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` + + `from the live counters in ${Date.now() - startedAt}ms — terminal, not a wait. If the ` + + `observed counts above look wrong for your data, run brain.repairIndex() to recount ` + + `from canonical.` + ) + } catch (error) { + prodLog.warn( + `${detail} The demotion's re-stamp FAILED (${(error as Error).message}) — the tear will ` + + `narrate again at the next open, which is the honest outcome; the store still serves ` + + `from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.` + ) + } + } + /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts index 98342884..2f01e935 100644 --- a/src/db/familyStamp.ts +++ b/src/db/familyStamp.ts @@ -12,9 +12,11 @@ * the verified surface is a small set of rollup invariants (entity/ * relationship counts) plus `sourceGeneration`. * - * `sourceGeneration` is the generation of the source-of-truth log this - * projection reflects — open-time coherence becomes a COMPARISON (stamp vs - * log head), not a walk: + * `sourceGeneration` is the COMMITTED generation of the source-of-truth log + * this projection reflects — never the allocated counter, which names a + * generation that may never commit (see {@link StampVerdict.torn}) — so + * open-time coherence becomes a COMPARISON (stamp vs committed head), not a + * walk: * * - equal + invariants hold → coherent, serve. * - behind → the projection missed the tail (crash between commit and stamp); @@ -24,6 +26,9 @@ * - invariants FAIL at equal generation → genuine incoherence: loud, and the * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * canonical walk) heals it. + * - AHEAD → a torn generation-log tail: the stamp's fsync outlived the log + * tail's. TERMINAL, never a wait — the generation the stamp names does not + * exist to arrive. * * Stamps are JSON on purpose — every incident gets debugged by reading a * stamp in a terminal. @@ -70,6 +75,12 @@ export type StampVerdict = | { state: 'coherent' } | { state: 'absent' } // legacy store — first stamp writes at the next flush | { state: 'behind'; stampSource: number; head: number } + /** + * TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the + * store's committed watermark can no longer show. TERMINAL — there is no + * generation to wait for, so the open demotes (or refuses) and never spins. + */ + | { state: 'torn'; stampSource: number; head: number } | { state: 'incoherent'; failures: string[] } | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence @@ -118,12 +129,15 @@ export function verifyFamilyStamp( ): StampVerdict { if (stamp === null) return { state: 'absent' } if (stamp.sourceGeneration > head) { - // A stamp AHEAD of the log claims state that never committed — the - // projection was stamped against truth that a crash rolled back. - return { - state: 'incoherent', - failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] - } + // A stamp AHEAD of committed truth witnesses a generation the store can no + // longer show: the stamp's fsync survived a crash that the log tail did + // not. This is the TORN GENERATION-LOG TAIL — its own class, never folded + // in with `incoherent` (a count that drifted at a generation both sides + // agree on), because the two have opposite cures: incoherence is recounted, + // a tear is DEMOTED. It is also terminal by construction — there is no + // generation the open can wait for, because the one the stamp names is + // gone. + return { state: 'torn', stampSource: stamp.sourceGeneration, head } } if (stamp.sourceGeneration < head) { return { state: 'behind', stampSource: stamp.sourceGeneration, head } diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts index deefc5e6..23cc0a15 100644 --- a/tests/integration/entity-tree-stamp.test.ts +++ b/tests/integration/entity-tree-stamp.test.ts @@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => { const invariants = (stamp.members as any).invariants expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) - expect(stamp.sourceGeneration).toBe(brain.generation()) + // THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the + // counter labelled the stamp with a generation a write in flight had merely + // claimed, so every crash inside a write window produced a spurious verdict + // at the next open (see the torn-tail pins below). + expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration()) expect(stamp.generation).toBeGreaterThanOrEqual(1) }) @@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => { expect(stillIncoherent).toEqual([]) }) + /** + * Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's + * committed watermark — the durable shape a torn generation-log tail leaves + * behind (the stamp's fsync outlived the tail's). Fabricated rather than + * crash-produced so the pin is deterministic; the seeded-SIGKILL lane + * (`scripts/crash-consistency.mjs` in the engine repo) produces the same + * shape from a real abrupt termination. + */ + const fabricateTear = (ahead: number): FamilyStamp => { + const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`) + const zlib = require('node:zlib') + const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp + const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead } + fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn))) + return torn + } + + it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => { + for (let i = 0; i < 3; i++) + await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } }) + await brain.close() + const torn = fabricateTear(5) + + const warn = vi.spyOn(prodLog, 'warn') + const startedAt = Date.now() + brain = await open() + const openMs = Date.now() - startedAt + + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + // Narrated PRECISELY: both generations, the file, and the named cure. + expect(said).toContain(`source generation ${torn.sourceGeneration}`) + expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`) + expect(said).toContain(ENTITY_TREE_STAMP_PATH) + expect(said).toContain('DEMOTED') + expect(said).toMatch(/repairIndex\(\)/) + // Terminal, not a wait: the demotion is O(1) straight-line work, so a tear + // cannot turn an open into the 8-minute spin this class was reported as. + expect(openMs).toBeLessThan(30_000) + + // The store SERVES — a tear in a stamp never locks an owner out of the + // canonical tree the stamp merely describes. + expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3) + + // The demotion CONVERGED: the stamp now names committed truth, and the + // next open is quiet. A verdict that re-narrates every open is a wait + // wearing a different hat. + const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp + expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration()) + await brain.close() + const warn2 = vi.spyOn(prodLog, 'warn') + brain = await open() + expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([]) + }) + + it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => { + await brain.add({ data: 'ro', type: 'document', metadata: {} }) + await brain.close() + const torn = fabricateTear(3) + + const warn = vi.spyOn(prodLog, 'warn') + const reader: any = await Brainy.openReadOnly({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + expect(said).toContain('READ-ONLY') + expect(said).toContain('UNVERIFIED') + expect(said).toMatch(/repairIndex\(\)/) + await reader.close() + + // A reader never rewrites the store: read the bytes back off disk (not + // through a writer open, which would demote them) — the torn stamp is + // exactly as it was found. + const onDisk = JSON.parse( + require('node:zlib') + .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) + .toString('utf-8') + ) as FamilyStamp + expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration) + expect(onDisk.generation).toBe(torn.generation) + + brain = await open() + }) + it('the one verifier handles both member modes', () => { const rollup: FamilyStamp = { family: 'x', @@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => { stampSource: 5, head: 9 }) - expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head + // AHEAD is its own class — a torn generation-log tail, never folded in + // with `incoherent`: the two have opposite cures (recount vs demote). + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({ + state: 'torn', + stampSource: 5, + head: 3 + }) expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) const enumerated: FamilyStamp = { From 9a888c37e9ebec5573cd7ebd0764396f3a424de3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:13:42 -0700 Subject: [PATCH 36/42] fix(generations): a sealed segment may only declare the generations it holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis of the "packed history is damaged" narration that fires on every run of the affected stores. It is a WRITER defect, and the reader's refusal was the symptom rather than the cause. A sealed segment declares one contiguous range [firstGeneration, lastGeneration], and every reader treats that range as containment: coveringSegment is an interval test, hasGeneration returns true for anything inside it, and open() seeds committedRanges from it. repackHistory handed fold() a SPARSE batch. Three filters punch holes in its candidate list mid-run — a generation absent from committedRanges never appears, one still in the pending buffer is skipped, one whose tx.json will not read is skipped — and fold() then computed the range from the first and last survivor, claiming every generation in between. The next open merged that mis-declared range back into committedRanges, re-admitting the hole as committed history, so the following auto-compaction pass asked the packed tier for a frame that was never written and failed. Re-merged at every open, which is why it repeated on every run. Confirmed against a forensic fixture: generation directories 1..2503 present except exactly one, 1416; and its fact-log segment already showed the tell — seg-...1410.bfl declaring 1410..1940 (531 generations) while recording 530 facts. Three changes: - repackHistory folds each contiguous RUN as its own segment (`contiguousRuns`), so ranges describe exactly what the segments contain. - fold() REFUSES a non-contiguous batch, naming the gap and its width. The density law is now mechanical, so no future caller can reintroduce it. A refusal loses nothing: the generations stay live and readable. - Stores already carrying the damage heal instead of wedging. A segment whose declared span exceeds its frame count is SPARSE; `actualRanges()` reads the real generation list from its sidecar so open() never re-admits the holes, and readFrame reports such a hole as unpacked with a narration naming the segment, rather than throwing. A DENSE segment missing a frame is still loud damage — that one means the manifest and sidecar disagree. Pins: nine unit cases (refusal and its message, honest ranges for separately folded runs, a reconstructed pre-fix sparse segment serving its real frames while reporting holes as unpacked, holes excluded from actualRanges, and the dense-segment damage path still throwing) plus an end-to-end case that deletes a generation directory and drives the real sequence — ordinary close()-time repacking folds over the hole, then reopen and compact must both complete. Verified red without the fix: the segment declared an 11-generation span while holding 10 frames. --- src/db/generationSegments.ts | 119 +++++++++++++++++++- src/db/generationStore.ts | 66 +++++++++-- tests/integration/history-repacking.test.ts | 102 +++++++++++++++++ tests/unit/db/generation-segments.test.ts | 115 +++++++++++++++++++ 4 files changed, 389 insertions(+), 13 deletions(-) diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts index 0c14b60c..91451281 100644 --- a/src/db/generationSegments.ts +++ b/src/db/generationSegments.ts @@ -147,6 +147,60 @@ export class GenerationSegmentStore { return this.coveringSegment(gen) !== null } + /** + * @description True when `meta` declares more generations than it holds + * frames — a segment sealed by a writer that folded across a hole. The + * manifest records `frames` at fold time, so this is an O(1) comparison + * against the declared span and needs no I/O. + */ + private isSparse(meta: SegmentMeta): boolean { + return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames + } + + /** + * @description The generations this tier ACTUALLY holds, as coalesced + * ascending intervals — not what the segments declare. + * + * Dense segments (every one a current writer produces) contribute their + * declared range with no I/O. A SPARSE segment — one sealed before the + * density law was enforced, whose declared range spans generations it has + * no frame for — has its real generation list read from its sidecar and + * contributed instead, with the discrepancy narrated once. + * + * This is what keeps a store that already carries the damage from wedging. + * `open()` seeds `committedRanges` from these intervals, so a hole is never + * re-admitted as a committed generation, and the auto-compaction pass that + * used to fail on every run with "packed history is damaged" simply never + * asks for the missing frame. + * + * @returns Ascending, non-overlapping `[first, last]` intervals. + */ + async actualRanges(): Promise> { + const out: Array<[number, number]> = [] + for (const meta of this.manifest.segments) { + if (!this.isSparse(meta)) { + out.push([meta.firstGeneration, meta.lastGeneration]) + continue + } + const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames + prodLog.warn( + `[GenerationSegments] sealed segment ${meta.file} declares generations ` + + `${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` + + `frame(s) — ${missing} generation(s) in that span were never folded into it. ` + + `Serving the frames it actually holds; the declared span is not treated as ` + + `committed history. (Written by a pre-density-law writer that folded across a ` + + `gap; the segment itself is intact and no record is lost.)` + ) + const idx = await this.sidecarFor(meta) + for (const [gen] of idx.generations) { + const last = out[out.length - 1] + if (last !== undefined && gen === last[1] + 1) last[1] = gen + else out.push([gen, gen]) + } + } + return out + } + /** * Fold consecutive generations into ONE new sealed segment + sidecar and * append it to the manifest atomically. Caller guarantees: `gens` is @@ -164,6 +218,38 @@ export class GenerationSegmentStore { throw new Error('[GenerationSegments] fold() input must be strictly ascending') } } + // THE DENSITY LAW, MADE MECHANICAL. + // + // A sealed segment declares a CONTIGUOUS range [firstGeneration, + // lastGeneration] and every reader treats that range as containment: + // `coveringSegment` is an interval test, `hasGeneration` returns true for + // anything inside it, and `open()` seeds committedRanges from it. So a + // segment folded from a SPARSE input silently claims generations it does + // not hold, and the first read of one of those holes throws + // "inside sealed segment ... but has no frame — packed history is damaged". + // + // That is exactly how the damage was produced. `repackHistory` skipped + // generations mid-batch — ones absent from committedRanges, ones still in + // the pending buffer, ones whose tx.json would not read — and handed the + // survivors here, where the range was computed from the first and last of + // them. Worse, the mis-declared range was then merged back into + // committedRanges at the next open, which is what turned a quiet hole into + // a repeating auto-compaction failure on every subsequent run. + // + // Callers now split at discontinuities; this refusal is what keeps any + // future caller from reintroducing the class. A refusal here loses + // nothing — the generations stay in the live tier, readable, and the next + // pass folds them correctly. + for (let i = 1; i < gens.length; i++) { + if (gens[i].generation !== gens[i - 1].generation + 1) { + throw new Error( + `[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` + + `${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` + + `generation(s). A sealed segment declares a dense range, so folding a sparse ` + + `batch would claim generations it does not hold. Split the batch at the gap.` + ) + } + } const last = this.manifest.segments[this.manifest.segments.length - 1] if (last && gens[0].generation <= last.lastGeneration) { throw new Error( @@ -364,12 +450,37 @@ export class GenerationSegmentStore { return this.decodeFrame(payload) } } - // In the covering range but not present: the packed tier is dense by - // construction (fold packs every generation it is handed, including - // record-less ones) — absence inside a sealed range is damage. + // Inside the covering range but with no frame. Two very different causes, + // and conflating them is what made this class wedge every maintenance pass + // on the affected stores. + // + // (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than + // the span it declares. That segment was sealed by a writer that + // folded across a hole (the class this file's density law now bars). + // The segment is INTACT and nothing is lost; it simply never held this + // generation. Answering "not packed" is the honest answer, and it lets + // the caller's two-tier read decide what a genuinely absent generation + // means, instead of every compaction pass dying on a repeating throw. + // `actualRanges()` keeps such holes out of committedRanges at open, so + // in a healed store nobody asks this question in the first place. + // + // (2) A DENSE SEGMENT missing a frame it says it has — the manifest and + // the sidecar disagree about a segment that claims to be complete. + // That IS damage, and it stays loud. + if (this.isSparse(meta)) { + prodLog.warn( + `[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` + + `declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` + + `holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` + + `-generation span — it was sealed across a gap and never held this generation. ` + + `Reporting it as unpacked rather than as damage; no record is lost.` + ) + return null + } throw new Error( `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame — packed history is damaged` + `range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + + `span — the manifest and the sidecar disagree; packed history is damaged` ) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index fd052c31..da21dc61 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -96,6 +96,35 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' +/** + * @description Split an ascending list of fold candidates into maximal + * CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`. + * + * A sealed segment declares one dense range `[firstGeneration, + * lastGeneration]`, and every reader treats that range as containment. So a + * batch with a hole in it must never become one segment: it would claim a + * generation it does not hold, and the first read of that hole reports the + * packed history as damaged. One run, one segment — the ranges then describe + * exactly what the segments contain. + * + * @param gens - Fold candidates, strictly ascending by generation. + * @returns One array per contiguous run, in ascending order. Empty in, empty out. + */ +export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] { + const runs: FoldGeneration[][] = [] + let run: FoldGeneration[] = [] + for (const g of gens) { + const prev = run[run.length - 1] + if (prev !== undefined && g.generation !== prev.generation + 1) { + runs.push(run) + run = [] + } + run.push(g) + } + if (run.length > 0) runs.push(run) + return runs +} + /** * @description Phases of the {@link GenerationStore.commitTransaction} commit * protocol at which a test-only fault injector can simulate a process crash. @@ -784,9 +813,15 @@ export class GenerationStore { if (storageSupportsFactLog(this.storage)) { this.segments = new GenerationSegmentStore(this.storage) await this.segments.open() - const packedRanges = this.segments - .segments() - .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) + // ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law + // writer can declare a span wider than the frames it holds; seeding + // committedRanges from the declared span re-admits those holes as + // committed generations, and every later maintenance pass then asks for a + // frame that was never written. `actualRanges()` reads the real + // generation list from the sidecar for exactly those segments (and does + // no I/O for the dense ones, which is all of them on a healthy store). + const packedRanges = (await this.segments.actualRanges()) + .map((r): [number, number] => [r[0], Math.min(r[1], this.committed)]) .filter(([lo, hi]) => lo <= hi) if (packedRanges.length > 0) { // Merge packed (older) + live (newer) interval sets — both ascending; @@ -3121,13 +3156,26 @@ export class GenerationStore { foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) } if (foldInput.length === 0) continue - await segments.fold(foldInput) - segmentsCreated++ - // Segment + manifest durable → the live copies retire. - for (const g of foldInput) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + // SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three + // filters above punch holes in it: a generation missing from + // committedRanges never appears, one still in the pending buffer is + // skipped, and one whose tx.json will not read is skipped. A sealed + // segment declares a DENSE range, so folding across such a hole makes + // the segment claim a generation it does not hold; the next open + // merges that mis-declared range into committedRanges, and every + // subsequent auto-compaction pass then asks for the missing frame and + // fails with "packed history is damaged". Fold each contiguous RUN as + // its own segment instead — same bytes, honest ranges. + for (const run of contiguousRuns(foldInput)) { + if (deadline !== undefined && Date.now() >= deadline) break + await segments.fold(run) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of run) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + } + folded += run.length } - folded += foldInput.length } if (folded > 0) { prodLog.info( diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts index 2bcee038..bb07268d 100644 --- a/tests/integration/history-repacking.test.ts +++ b/tests/integration/history-repacking.test.ts @@ -16,6 +16,7 @@ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as path from 'node:path' import * as os from 'node:os' +import * as zlib from 'node:zlib' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { GenerationStore } from '../../src/db/generationStore.js' @@ -57,6 +58,107 @@ describe('history repacking — the two-tier lifecycle', () => { } }) + /** + * THE HOLE, END TO END — the shape a real store carries. + * + * A forensic fixture was measured with generation directories 1..2503 + * present except for exactly one: 1416. Its fact-log segment already showed + * the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration + * 1940 (531 generations) while recording only 530 facts. + * + * Before the fix, repacking such a store folded ACROSS that hole: the batch + * skipped 1416 (no readable delta) and the sealed segment declared a range + * spanning it anyway. The next open merged that declared range back into + * committedRanges, re-admitting 1416 as committed history, and every + * subsequent auto-compaction pass then asked the packed tier for a frame + * that was never written — producing, on EVERY run, the non-fatal narration + * + * Auto-compaction of generational history failed (non-fatal): generation + * N is inside sealed segment seg-....bgs's declared range but has no frame + * — packed history is damaged + * + * This pin removes a generation directory to make the same hole, then + * requires repack + reopen + compaction to complete cleanly. + */ + it('a missing generation directory does not poison the packed tier', async () => { + const dir = tempDir() + // `retention: 'all'` throughout: close() otherwise auto-compacts the + // history away, and this pin needs the cold generations still on disk so + // there is something to punch a hole in. The live window stays at its + // production default for the build phase, so nothing folds yet. + const archival = async (): Promise => { + const b = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + embeddingFunction: stub, + retention: 'all' + }) + await b.init() + return b + } + const brain = await archival() + + const id = await brain.add({ + data: 'holed-entity', + type: NounType.Document, + metadata: { v: 0 } + }) + // One flush per update: single-op writes coalesce inside a flush window, + // so a history deep enough to have a middle needs the windows separated. + for (let v = 1; v <= 12; v++) { + await brain.update({ id, metadata: { v } }) + await brain.flush() + } + await brain.close() + + // Punch the hole: delete ONE generation directory in the middle of the + // cold range, exactly as the real store presents it. + const genRoot = path.join(dir, '_generations') + const numeric = fs + .readdirSync(genRoot, { withFileTypes: true }) + .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)) + .map((e) => Number(e.name)) + .sort((a, b) => a - b) + expect(numeric.length).toBeGreaterThan(6) + const victim = numeric[Math.floor(numeric.length / 2)] + fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true }) + + // Now shrink the live window and reopen. close() repacks automatically + // (brainy.ts phase 0b), so this is the production sequence exactly: a + // store with a hole in its history gets folded by ordinary housekeeping, + // with nobody asking for it. + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 + const reopened = await archival() + const result = await reopened.repackHistory() + expect(result.foldedGenerations).toBeGreaterThan(0) + + const segDir = path.join(dir, SEGMENTS_PREFIX) + const manifestPath = ['manifest.json', 'manifest.json.gz'] + .map((f) => path.join(segDir, f)) + .find((p) => fs.existsSync(p))! + const raw = manifestPath.endsWith('.gz') + ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8') + : fs.readFileSync(manifestPath, 'utf8') + const manifest = JSON.parse(raw) as { + segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }> + } + + // THE LAW: every sealed segment declares exactly as many generations as it + // holds frames, and none of them spans the victim. + for (const s of manifest.segments) { + expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames) + expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false) + } + + await reopened.close() + + // And the pass that used to fail on every run now completes: reopen (which + // re-seeds committedRanges from the packed tier) then compact history. + const third = await openBrain(dir) + await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined() + await third.close() + }) + it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 const dir = tempDir() diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts index 27ab85cb..f16e67b3 100644 --- a/tests/unit/db/generation-segments.test.ts +++ b/tests/unit/db/generation-segments.test.ts @@ -147,4 +147,119 @@ describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) + + // ========================================================================== + // THE DENSITY LAW + // ========================================================================== + // + // A sealed segment declares a CONTIGUOUS range and every reader treats that + // range as containment. Folding a sparse batch therefore makes the segment + // claim generations it does not hold — and because `open()` merges declared + // ranges back into committedRanges, the hole is re-admitted as committed + // history and every later maintenance pass fails asking for a frame that was + // never written. That is the "generation N is inside sealed segment + // seg-....bgs's declared range but has no frame — packed history is damaged" + // narration seen on every run of the affected stores. + + it('fold REFUSES a batch with a hole — a dense range may not be declared over sparse input', async () => { + await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow( + /not contiguous: 2 → 4 skips 1 generation/ + ) + // The refusal loses nothing: no segment was sealed, so the generations + // stay in the live tier and the next pass folds them correctly. + expect(store.segments()).toHaveLength(0) + expect(store.hasGeneration(1)).toBe(false) + }) + + it('a wider gap names how many generations it would have swallowed', async () => { + await expect(store.fold([gen(10), gen(20)])).rejects.toThrow( + /not contiguous: 10 → 20 skips 9 generation\(s\)/ + ) + }) + + it('two contiguous runs folded separately declare honest ranges', async () => { + // What the caller now does instead of folding across the gap. + const a = await store.fold([gen(1), gen(2), gen(3)]) + const b = await store.fold([gen(7), gen(8)]) + expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) + expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 }) + // The gap is honestly outside the packed tier. + for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false) + for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true) + expect(await store.actualRanges()).toEqual([ + [1, 3], + [7, 8] + ]) + }) + + it('actualRanges() is exact and I/O-free for dense segments', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + // Adjacent dense segments each contribute their declared range. + expect(await store.actualRanges()).toEqual([ + [1, 2], + [3, 4] + ]) + }) + + // ---- pre-existing damage: a store sealed by the old writer ---------------- + + /** + * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and + * sidecar for a contiguous run, then rewrite the manifest so the segment + * declares a wider range than the frames it holds. This reproduces on disk + * exactly what the affected stores carry, without needing the old code. + */ + const sealSparseSegment = async (): Promise => { + await store.fold([gen(1), gen(2), gen(3)]) + const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any + // Declare 1..5 while holding frames for 1..3 — generations 4 and 5 become + // holes inside a sealed range. + manifest.segments[0].lastGeneration = 5 + await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest) + } + + it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + + // The frames it really holds still serve, byte-faithfully. + expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002) + expect(await reopened.readRecords(3)).toHaveLength(2) + + // The holes answer "not packed" instead of throwing. This is the fix for + // the wedge: the old reader threw here on EVERY maintenance pass. + expect(await reopened.readDelta(4)).toBeNull() + expect(await reopened.readRecords(5)).toBeNull() + }) + + it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + // Declared 1..5; actually holds 1..3. The store seeds committedRanges from + // THIS, so generations 4 and 5 never become committed history again. + expect(await reopened.actualRanges()).toEqual([[1, 3]]) + }) + + it('a DENSE segment missing a frame is still loud damage', async () => { + // The other side of the branch: when the manifest claims a complete span, + // a missing frame means the manifest and sidecar disagree — real damage, + // and it must not be quietly downgraded to "unpacked". + await store.fold([gen(1), gen(2), gen(3)]) + const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` + const raw = (await storage.readRawBytes(idxPath))! + const { decode, encode } = await import('@msgpack/msgpack') + const idx = decode(raw) as any + // Drop generation 2's entry while the manifest still declares 3 frames. + idx.generations = idx.generations.filter(([g]: [number]) => g !== 2) + await storage.writeRawBytes(idxPath, encode(idx)) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + await expect(reopened.readDelta(2)).rejects.toThrow( + /manifest and the sidecar disagree; packed history is damaged/ + ) + }) }) From 655aa13ea79e23927cde7fd47ab13b505cb042d9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:30:46 -0700 Subject: [PATCH 37/42] =?UTF-8?q?build(release):=20the=20docs-push=20step?= =?UTF-8?q?=20retires=20=E2=80=94=20this=20engine=20documents=20itself=20i?= =?UTF-8?q?n=20its=20own=20repository?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one-doc-set ruling (2026-08-31) gives soulcraft.com/docs to the paid product alone; the site serves redirects for the slugs this rail used to push. The push script stays in the tree as history; the rail stops calling it. --- scripts/release.sh | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 67ad1053..5d434320 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -248,17 +248,12 @@ else echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" fi -# Step 12: Push public docs to the soulcraft.com docs ingest door -# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when -# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — -# that already happened) when a push errors, so the docs site never -# silently trails npm. -echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" -if node scripts/push-docs.js; then - echo -e "${GREEN}✅ Docs push step done${NC}\n" -else - echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" -fi +# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): +# soulcraft.com/docs carries the paid product's documentation only. This +# engine's documentation home is THIS repository — README and docs/ — and the +# site serves 301s for the slugs this rail used to push. The push script stays +# in the tree for history; the rail no longer calls it. +echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" From e64e2bc17580737a0b7a63e2af8ea6b6c527278a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 12:04:29 -0700 Subject: [PATCH 38/42] =?UTF-8?q?docs(releases):=20the=20release-notes=20d?= =?UTF-8?q?oor=20=E2=80=94=20owner-language=20notes=20for=20both=20engines?= =?UTF-8?q?,=20backfilled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fleet's releases wall reads one public URL per product. These files are that door for Brainy and Open Brainy: newest first, honest history from the changelog, one entry appended by every release from here on. --- releases/brainy.json | 52 +++++++++++++++++++++++ releases/open-brainy.json | 87 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 releases/brainy.json create mode 100644 releases/open-brainy.json diff --git a/releases/brainy.json b/releases/brainy.json new file mode 100644 index 00000000..17217a6e --- /dev/null +++ b/releases/brainy.json @@ -0,0 +1,52 @@ +{ + "product": "brainy", + "entries": [ + { + "version": "11.0.3", + "date": "2026-09-01", + "headline": "The embedding upgrade ceremony runs on every brain", + "items": [ + "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", + "A one-fix release; nothing else changed." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.2", + "date": "2026-08-31", + "headline": "One embedding quality everywhere, 3–4× faster imports", + "items": [ + "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", + "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", + "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.1", + "date": "2026-08-31", + "headline": "Deletes inside transactions are safe", + "items": [ + "Deleting relations inside a transact() no longer corrupts index bookkeeping.", + "A store that deletes its last relation keeps serving instead of refusing." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.0", + "date": "2026-08-28", + "headline": "One install, one engine — Brainy", + "items": [ + "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", + "A missing native build refuses loudly with its cures named; nothing falls back silently.", + "Stores open in place — no migration." + ], + "url": null, + "thumb": null + } + ], + "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." +} diff --git a/releases/open-brainy.json b/releases/open-brainy.json new file mode 100644 index 00000000..21014f5b --- /dev/null +++ b/releases/open-brainy.json @@ -0,0 +1,87 @@ +{ + "product": "open-brainy", + "entries": [ + { + "version": "10.4.6", + "date": "2026-08-31", + "headline": "Transactions cross the index seam safely", + "items": [ + "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", + "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", + "thumb": null + }, + { + "version": "10.4.5", + "date": "2026-08-31", + "headline": "Recovery tells the truth, docs live at home", + "items": [ + "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", + "A sealed segment declares only the generations it actually holds.", + "The engine's documentation now publishes from its own repository." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", + "thumb": null + }, + { + "version": "10.4.4", + "date": "2026-08-28", + "headline": "Faster opens, quieter idle", + "items": [ + "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", + "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", + "A slow open now names the exact step it is in, so operators see what is being paid and why." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", + "thumb": null + }, + { + "version": "10.4.3", + "date": "2026-08-27", + "headline": "Open Brainy, under its own name", + "items": [ + "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", + "No code changes; your imports change once and everything else stays put." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", + "thumb": null + }, + { + "version": "10.4.2", + "date": "2026-08-27", + "headline": "Vectors that lie are refused, counts that drift are caught", + "items": [ + "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", + "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", + "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", + "thumb": null + }, + { + "version": "10.4.1", + "date": "2026-08-26", + "headline": "Writes that change nothing cost nothing", + "items": [ + "The read gate is per index family, and a write carrying unchanged data never re-embeds.", + "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", + "thumb": null + }, + { + "version": "10.4.0", + "date": "2026-08-26", + "headline": "Repair routing, the vector ledger, and honest empties", + "items": [ + "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", + "An empty string is real data, not a missing field.", + "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", + "thumb": null + } + ], + "history": "Earlier releases are recorded in CHANGELOG.md in this repository." +} From f097cbf6f29cad03a34ca0c31c64fd86b53c9fdb Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 13:49:26 -0700 Subject: [PATCH 39/42] =?UTF-8?q?docs(releases):=20the=2010.4.7=20note=20?= =?UTF-8?q?=E2=80=94=20count=20ledgers=20can=20no=20longer=20race=20themse?= =?UTF-8?q?lves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/open-brainy.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index 21014f5b..e1dc83ce 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,17 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.7", + "date": "2026-09-01", + "headline": "Count ledgers can no longer race themselves", + "items": [ + "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", + "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", + "thumb": null + }, { "version": "10.4.6", "date": "2026-08-31", From 7ab670b525d62c86d7a39f81acdb1384fe2928ba Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 13:55:27 -0700 Subject: [PATCH 40/42] =?UTF-8?q?docs(releases):=20the=2011.0.4=20note=20?= =?UTF-8?q?=E2=80=94=20millisecond=20closes,=20storm-free=20rebuilds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/brainy.json b/releases/brainy.json index 17217a6e..c6c664a9 100644 --- a/releases/brainy.json +++ b/releases/brainy.json @@ -1,6 +1,18 @@ { "product": "brainy", "entries": [ + { + "version": "11.0.4", + "date": "2026-09-01", + "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", + "items": [ + "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", + "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", + "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." + ], + "url": null, + "thumb": null + }, { "version": "11.0.3", "date": "2026-09-01", From 297a3d76575771f60518a813b2c023e68bc9d707 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 08:30:01 -0700 Subject: [PATCH 41/42] =?UTF-8?q?docs(releases):=20the=2010.4.9=20note=20?= =?UTF-8?q?=E2=80=94=20graph-first=20finds,=20honest=20verb=20arrays,=20bo?= =?UTF-8?q?unded=20recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/open-brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index e1dc83ce..582f4847 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,18 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.9", + "date": "2026-09-02", + "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", + "items": [ + "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", + "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", + "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", + "thumb": null + }, { "version": "10.4.7", "date": "2026-09-01", From 4f1e27c9a089f5dc5d3b20f7ba9fc52384ee0e28 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 08:32:21 -0700 Subject: [PATCH 42/42] =?UTF-8?q?docs(releases):=20the=2011.0.5=20note=20?= =?UTF-8?q?=E2=80=94=20graph-first=20finds=20in=20production,=20bounded=20?= =?UTF-8?q?recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/brainy.json b/releases/brainy.json index c6c664a9..8f61c7f2 100644 --- a/releases/brainy.json +++ b/releases/brainy.json @@ -1,6 +1,18 @@ { "product": "brainy", "entries": [ + { + "version": "11.0.5", + "date": "2026-09-02", + "headline": "Graph-first finds in production, and opens that stop rescanning history", + "items": [ + "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", + "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", + "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." + ], + "url": null, + "thumb": null + }, { "version": "11.0.4", "date": "2026-09-01",