From 7b67db4d0c2f89468ea397ddd57c67dee380db9c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 12 Aug 2026 15:57:19 -0700 Subject: [PATCH 001/102] =?UTF-8?q?feat(query):=20the=20sparse-store=20cut?= =?UTF-8?q?=20=E2=80=94=20where=20on=20a=20never-carried=20field=20serves?= =?UTF-8?q?=20operator=20truth,=20never=20a=20refusal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A first adopter's namespace migration went 341 red on one class: the never-carried-field refusal firing on CORRECT filters against fresh and sparse stores — a freshly provisioned tenant refused its own first filtered read, with the did-you-mean built for typos firing hardest on day-one stores where nothing is wrong. The ruled cut: a WHERE filter naming a field no row carries is SERVED OPERATOR-TRUTHFULLY — eq/in/range/contains answer [] (nothing carries it, nothing matches); ne and exists:false answer ALL rows (the equally true complement — a blanket empty here would be silently wrong, which is why the simpler cut was rejected); exists:true answers []. Served from the field registry, with the did-you-mean demoted to a once-per-field WARN. orderBy and genuinely ambiguous addresses KEEP their hard typed refusals: no truthful order exists over an uncarried field, and ambiguity is a contract error while absence is data. Mechanics: the negative operator absorbs the FIELD_NOT_INDEXED throw as its empty exclude set (the clause-level catch correctly zeroes positive operators only); the egress matcher already agreed. Plus the provider-seam belt: a field refusal thrown by a replacement metadata manager is normalized to THIS package's UnresolvableFieldError at every filter call site — one class identity for consumers, instanceof works (a first adopter's cross-package finding). Conformance: tests/conformance/sparse-store-cut.test.ts — the shared operator rows both engines run (positive-empty, negative-all, fresh-tenant day-one, orderBy refusal kept, compound composition). Gates: unit 2065/2065 · integration 832 · conformance 36/36. --- src/brainy.ts | 36 +++++++-- src/db/fieldAddressing.ts | 22 ++++++ src/utils/metadataIndex.ts | 53 ++++++++++++- tests/conformance/sparse-store-cut.test.ts | 82 ++++++++++++++++++++ tests/unit/test-suite-coverage-guard.test.ts | 3 + 5 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 tests/conformance/sparse-store-cut.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index a0f06931..785d10e5 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -197,6 +197,7 @@ import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' import { assessIndexReadiness } from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' +import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { readLogAuthority, runLogCompletenessOracle, @@ -4107,7 +4108,7 @@ export class Brainy implements BrainyInterface { const probeServes = async (): Promise => { try { - const ids = await this.metadataIndex.getIdsForFilter({ [p.field]: p.value }) + const ids = await this.filterIdsBelted({ [p.field]: p.value }) return ids.includes(p.id) } catch { // FIELD_NOT_INDEXED for a field a persisted entity actually holds is @@ -6447,7 +6448,7 @@ export class Brainy implements BrainyInterface { // 'visibility' key would address the USER's metadata bag under the // field-addressing law and silently hide nothing (VFS/system entities // would leak into every default read). - const ids = await this.metadataIndex.getIdsForFilter({ + const ids = await this.filterIdsBelted({ 'system.visibility': excluded.length === 1 ? excluded[0] : { oneOf: excluded } }) return new Set(ids) @@ -6655,7 +6656,7 @@ export class Brainy implements BrainyInterface { // offset stays 0 because the visibility filter + slice happen here. The JS // index ignores the bound and returns all matches (behaviour unchanged). const pageEnd = (params.offset || 0) + (params.limit || 10) + hiddenIds.size - filteredIds = await this.metadataIndex.getIdsForFilter(filter, { limit: pageEnd, offset: 0 }) + filteredIds = await this.filterIdsBelted(filter, { limit: pageEnd, offset: 0 }) } // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact. @@ -6727,7 +6728,7 @@ export class Brainy implements BrainyInterface { // filter returns nothing from getIdsForFilter, so the unfiltered case below uses // getNouns instead (it returns all nouns, including their visibility). if (Object.keys(filter).length > 0) { - let filteredIds = await this.metadataIndex.getIdsForFilter(filter) + let filteredIds = await this.filterIdsBelted(filter) // Visibility hard filter — drop hidden ids BEFORE pagination. if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) const pageIds = filteredIds.slice(offset, offset + limit) @@ -6778,7 +6779,7 @@ export class Brainy implements BrainyInterface { if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { preResolvedFilter = this.buildMetadataFilter(params) - preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter) + preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter) // Visibility hard filter — restrict the HNSW candidate set to non-hidden ids. if (hiddenIds.size > 0) { @@ -11548,6 +11549,27 @@ export class Brainy implements BrainyInterface { * console.log(`Lazy rebuild completed: ${status.lazyRebuildCompleted}`) * ``` */ + + /** + * The provider-seam belt for filter reads: whatever manager serves + * getIdsForFilter (the JS twin or a native replacement), a field refusal + * crossing this seam is normalized to BRAINY'S UnresolvableFieldError — + * one class identity for consumers, never a foreign twin that fails + * instanceof. All other errors pass through untouched. + */ + private async filterIdsBelted( + filter: unknown, + opts?: { limit?: number; offset?: number } + ): Promise { + try { + return await this.metadataIndex.getIdsForFilter(filter, opts) + } catch (err) { + const normalized = asBrainyFieldRefusal(err) + if (normalized) throw normalized + throw err + } + } + async getIndexStatus(): Promise<{ initialized: boolean lazyRebuildCompleted: boolean @@ -12145,7 +12167,7 @@ export class Brainy implements BrainyInterface { } } - const filteredIds = await this.metadataIndex.getIdsForFilter(filter) + const filteredIds = await this.filterIdsBelted(filter) return filteredIds.length } @@ -12217,7 +12239,7 @@ export class Brainy implements BrainyInterface { } } - const filteredIds = await this.metadataIndex.getIdsForFilter(filterObj) + const filteredIds = await this.filterIdsBelted(filterObj) // Stream filtered entities in batches for memory efficiency const batchSize = 100 diff --git a/src/db/fieldAddressing.ts b/src/db/fieldAddressing.ts index 21689319..da04e74c 100644 --- a/src/db/fieldAddressing.ts +++ b/src/db/fieldAddressing.ts @@ -261,6 +261,28 @@ export function buildUnresolvableMessage( * the fix ships inside the error. Thrown by the query layer with index * knowledge, never by the pure parser. */ +/** + * Cross-package identity normalizer (the seam belt): the native accelerator + * throws ITS OWN UnresolvableFieldError class, which fails `instanceof` + * against this package's export — consumers were forced to match by name. + * Every provider-boundary catch routes suspected field-refusals through + * here: a foreign refusal (matched by name, duck fields tolerated) is + * rethrown as THIS package's class, so exactly one identity ever reaches + * consumers. Anything else returns null (caller rethrows the original). + */ +export function asBrainyFieldRefusal(err: unknown): UnresolvableFieldError | null { + if (err instanceof UnresolvableFieldError) return err + const e = err as { name?: string; message?: string; raw?: string; kind?: string } | null + if (e && e.name === 'UnresolvableFieldError') { + return new UnresolvableFieldError( + e.raw ?? 'unknown-field', + (e.kind as FieldAddressKind) ?? 'entity', + e.message + ) + } + return null +} + export class UnresolvableFieldError extends Error { public readonly raw: string public readonly kind: FieldAddressKind diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 894f3fd3..13cf3bb4 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1908,6 +1908,35 @@ export class MetadataIndexManager implements MetadataIndexProvider { * index (early-stop at `offset+limit`); the JS index returns ALL matches and lets * the caller window them, so `_opts` is intentionally ignored here. */ + /** Once-per-field throttle for the sparse-store did-you-mean WARN. */ + private readonly warnedNeverCarried = new Set() + + /** + * THE SPARSE-STORE CUT (ruled 2026-08-12): a WHERE filter naming a field + * no row carries is SERVED OPERATOR-TRUTHFULLY (eq/range/contains → []; + * ne/exists:false → all rows; exists:true → []) — the JS evaluator below + * already computes exactly these truths via complements — with the + * did-you-mean demoted to this throttled WARN. A fresh store's first + * filtered read is a correct empty answer, never a refusal. orderBy and + * ambiguous addresses KEEP their hard refusals (no truthful order + * exists; ambiguity is a contract error — absence is data). + */ + /** Is this field known to the index at all (any row ever carried it)? */ + private fieldRegistryHas(field: string): boolean { + return this.fieldStats.has(field) + } + + private warnNeverCarriedOnce(field: string): void { + if (this.warnedNeverCarried.has(field)) return + this.warnedNeverCarried.add(field) + prodLog.warn( + `[MetadataIndex] filter names field '${field}' which no row carries — ` + + `serving the operator-truthful answer (empty for positive matches; ` + + `the complement for ne/exists:false). If this is a typo, check the ` + + `field name; refusals remain on orderBy.` + ) + } + async getIdsForFilter(filter: any, _opts?: { limit?: number; offset?: number }): Promise { if (!filter || Object.keys(filter).length === 0) { return [] @@ -1984,6 +2013,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { const address = parseFieldAddress(rawField, 'entity') const field = address.scope === 'system' ? `system.${address.field}` : address.field + // Sparse-store cut: a user field no row carries serves operator-truth + // below (the evaluators' complements are already correct) — announce + // it once so a typo is findable without breaking a fresh store. + if ( + address.scope !== 'system' && + !(this.columnStore && this.columnStore.hasField(field)) && + !this.fieldRegistryHas(field) + ) { + this.warnNeverCarriedOnce(field) + } + let fieldResults: string[] = [] try { @@ -2022,7 +2062,18 @@ export class MetadataIndexManager implements MetadataIndexProvider { // complement as a bitmap difference over the int-id universe rather // than materializing the whole corpus as UUID strings to filter it. const excludeInts: number[] = [] - for (const uuid of await this.getIds(field, operand)) { + // Sparse-store truth: a never-carried field has NOTHING to + // exclude — the complement of nothing is EVERYTHING. getIds + // throws FIELD_NOT_INDEXED there; the clause-level catch + // would wrongly zero this NEGATIVE operator, so absorb it + // here as the empty exclude set (the ruled operator-truth). + let neMatches: string[] = [] + try { + neMatches = await this.getIds(field, operand) + } catch { + neMatches = [] + } + for (const uuid of neMatches) { const intId = this.idMapper.getInt(uuid) if (intId !== undefined) excludeInts.push(intId) } diff --git a/tests/conformance/sparse-store-cut.test.ts b/tests/conformance/sparse-store-cut.test.ts new file mode 100644 index 00000000..8a06c98c --- /dev/null +++ b/tests/conformance/sparse-store-cut.test.ts @@ -0,0 +1,82 @@ +/** + * @module tests/conformance/sparse-store-cut + * @description THE SPARSE-STORE CUT (ruled 2026-08-12) — the shared + * conformance rows both engines run: a WHERE filter naming a field NO row + * carries is SERVED OPERATOR-TRUTHFULLY, never refused: + * eq / in / range / contains → [] (nothing carries it → nothing matches) + * ne / exists:false → ALL rows (equally true — blanket-empty here + * would be the outlawed silent wrong) + * exists:true → [] + * orderBy on an unresolvable field KEEPS the hard refusal (no truthful + * order exists). The did-you-mean demotes to a throttled WARN on the serve. + * A fresh tenant's first filtered read is a correct empty answer — the + * 341-red first-adopter class, closed. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy, UnresolvableFieldError } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) +}) + +async function corpus(): Promise<{ brain: Brainy; ids: string[] }> { + const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) + await b.init() + brains.push(b) + const ids: string[] = [] + for (let i = 0; i < 4; i++) { + ids.push( + await b.add({ data: `row ${i}`, type: NounType.Document, metadata: { carried: i } }) + ) + } + return { brain: b, ids } +} + +describe('sparse-store cut — operator-truthful serve on never-carried fields', () => { + it('positive matches serve EMPTY: eq, in, range, contains', async () => { + const { brain } = await corpus() + expect(await brain.find({ where: { ghost: 'x' }, limit: 10 })).toEqual([]) + expect(await brain.find({ where: { ghost: { in: ['a', 'b'] } }, limit: 10 })).toEqual([]) + expect(await brain.find({ where: { ghost: { gt: 5 } }, limit: 10 })).toEqual([]) + expect(await brain.find({ where: { ghost: { exists: true } }, limit: 10 })).toEqual([]) + }) + + it('negative matches serve ALL rows: ne and exists:false (the truth, not blanket-empty)', async () => { + const { brain, ids } = await corpus() + const ne = await brain.find({ where: { ghost: { ne: 'x' } }, limit: 10 }) + expect(ne.map((r) => r.id).sort()).toEqual([...ids].sort()) + const absent = await brain.find({ where: { ghost: { exists: false } }, limit: 10 }) + expect(absent.map((r) => r.id).sort()).toEqual([...ids].sort()) + }) + + it('the fresh-tenant day-one shape: an EMPTY store answers its first filtered read with [], never a refusal', async () => { + const b = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) + await b.init() + brains.push(b) + expect(await b.find({ where: { status: 'open' }, limit: 50 })).toEqual([]) + expect(await b.find({ where: { date: { gte: '2026-01-01' } }, limit: 50 })).toEqual([]) + }) + + it('orderBy on an unresolvable field KEEPS the typed refusal', async () => { + const { brain } = await corpus() + await expect( + brain.find({ where: { carried: { gte: 0 } }, orderBy: 'system.notAScalar', limit: 10 }) + ).rejects.toThrow(UnresolvableFieldError) + }) + + it('compound filters: the never-carried clause composes truthfully with carried clauses', async () => { + const { brain, ids } = await corpus() + // carried>=2 AND ghost ne 'x' → the carried>=2 rows (ne-clause = all). + const both = await brain.find({ + where: { carried: { gte: 2 }, ghost: { ne: 'x' } }, + limit: 10 + }) + expect(both.map((r) => r.id).sort()).toEqual([ids[2], ids[3]].sort()) + // carried>=2 AND ghost eq 'x' → [] (eq-clause empties the intersection). + expect( + await brain.find({ where: { carried: { gte: 2 }, ghost: 'x' }, limit: 10 }) + ).toEqual([]) + }) +}) diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index 4b078146..c43b2cf2 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -37,6 +37,9 @@ const MANUAL_ONLY = new Set([ // (byte + fold digests) — runs in the explicit conformance gate stage, // same invocation family as the other conformance suites. 'tests/conformance/golden-log-fold.test.ts', + // The sparse-store cut's shared operator rows (both engines run these): + // explicit conformance-gate invocation, like its siblings. + 'tests/conformance/sparse-store-cut.test.ts', 'tests/api/performance-benchmarks.test.ts', 'tests/critical-neural-validation.test.ts', 'tests/critical-performance-benchmark.test.ts', From cbe34d115e9b364c75382c659e51559a8d262dd7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 12 Aug 2026 16:09:48 -0700 Subject: [PATCH 002/102] =?UTF-8?q?fix(log):=20pad-frame=20construction=20?= =?UTF-8?q?is=20total;=20the=20at-ack=20sync-failure=20compensation=20spli?= =?UTF-8?q?ts=20by=20phase=20=E2=80=94=20a=20production=20adoption's=20two?= =?UTF-8?q?=20write-path=20defects,=20cured=20at=20their=20roots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adopter's full suite found two v2 write-path defects on fresh brains, reproduced with stacks; both cured and both pinned with their exact production shapes: 1. PAD-FRAME CONSTRUCTIBILITY: a single msgpack bin filler steps its header by one byte at each size class (bin8→bin16→bin32), leaving one unreachable payload size per boundary — the sealer requested a 291-byte pad, the encoder threw 'not constructible', and sync() died whole. Construction is now TOTAL: the class-boundary holes bridge with a trailing fixint beside the bin ({bin(n)} ∪ {bin(n)+fixint} covers every size ≥ minimum). Pinned exhaustively: every size from the minimum through a full sector plus boundary spill constructs byte-exact and decodes as reader-invisible filler. 2. THE NON-MONOTONIC REFUSAL LOOP: the append-failure compensation rewound the generation counter on ANY throw — including a covering SYNC failure after a SUCCESSFUL append. The log carried generation N while the counter re-minted N, and every later append refused 'non-monotonic (N ≤ head N)' — the write path wedged in a refusal loop through deferred-embed retries and flush backoff. The compensation now splits by phase: an append failure (log never took the fact) fully compensates — un-buffer and rewind; a sync failure after append earns the rewind ONLY if the appended fact is provably dropped, otherwise the generation stays consumed and buffered — the counter never re-mints a number the log may carry. Pinned: an injected one-shot sync failure fails its write loudly and the very next write mints fresh and succeeds, with the log scanning strictly ascending end to end. Also probed against the adopter's carried report: the 9.0 vfs.rename stale-ghost shape does NOT reproduce on this head (old path cleanly unresolvable on exists/stat/readdir after rename). Gates: unit 2067/2067 (160 files) · integration 833 (97 files) · conformance 36/36. --- src/db/factLogFormat.ts | 24 ++++++ src/db/generationStore.ts | 56 +++++++++---- .../sync-fail-compensation.test.ts | 78 +++++++++++++++++++ tests/unit/db/pad-frame-total.test.ts | 29 +++++++ 4 files changed, 173 insertions(+), 14 deletions(-) create mode 100644 tests/integration/sync-fail-compensation.test.ts create mode 100644 tests/unit/db/pad-frame-total.test.ts diff --git a/src/db/factLogFormat.ts b/src/db/factLogFormat.ts index 0ac93e1f..d5492da8 100644 --- a/src/db/factLogFormat.ts +++ b/src/db/factLogFormat.ts @@ -1204,6 +1204,30 @@ function buildPadFrame(totalBytes: number): Uint8Array { fillerLength += diff if (fillerLength < 0) break } + if (!converged) { + // Class-boundary holes: a single bin filler steps its header by one + // byte at each msgpack size class (bin8→bin16→bin32), leaving exactly + // one unreachable payload size per boundary (the 291-byte production + // case). Bridge with a trailing fixint (+1 byte) beside the bin — + // {bin(n)} ∪ {bin(n) + fixint} covers every size ≥ minimum. + let bridged = Math.max(0, targetPayload - payload.length - 2) + for (let i = 0; i < 8; i++) { + const candidate = attempt([ + LOG_RECORD_TYPES.PAD, + LOG_RECORD_VERSION, + new Uint8Array(bridged), + 0 + ]) + const diff = targetPayload - candidate.length + if (diff === 0) { + payload = candidate + converged = true + break + } + bridged += diff + if (bridged < 0) break + } + } if (!converged) { throw new Error(`fact log v2: a pad frame of ${totalBytes} bytes is not constructible`) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 6922da6d..8f3bf625 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -1555,6 +1555,21 @@ export class GenerationStore { // the log's group-commit (many concurrent writers share ONE sync) — // an acked write's fact survives power loss, by contract. if (this.factLog) { + // TWO PHASES, TWO DISTINCT COMPENSATIONS (a production adoption + // proved the difference the hard way): rewinding the counter after + // a SUCCESSFUL append re-mints the same generation and every later + // append refuses non-monotonic — the write path wedges in a refusal + // loop. The counter may only rewind when the log provably does NOT + // carry the generation. + const unbuffer = (): void => { + this.pendingBuffer.delete(gen) + const idx = this.pendingGens.lastIndexOf(gen) + if (idx !== -1) this.pendingGens.splice(idx, 1) + this.invalidateChains() + } + // Phase 1 — APPEND. Failure = the log never took the fact: full + // compensation (un-buffer + counter rewind); a rejected write must + // not commit, and the next mint may safely reuse the number. try { await this.factLog.append( await this.buildCommitFact({ @@ -1565,24 +1580,37 @@ export class GenerationStore { ...(args.records && args.records.length > 0 ? { records: args.records } : {}) }) ) - if (this.logDurability === 'at-ack') { - await this.factLog.ensureSynced() - } } catch (err) { - // A rejected write must NOT commit: the generation was buffered - // before the append, so un-buffer it and return the counter - // reservation — otherwise the next flush would durably commit a - // generation with NO fact, a silent log gap a later replay would - // turn into loss. Canonical bytes from execute() remain as an - // uncommitted orphan — identical to a crash at this point; never - // a torn committed state. - this.pendingBuffer.delete(gen) - const idx = this.pendingGens.lastIndexOf(gen) - if (idx !== -1) this.pendingGens.splice(idx, 1) - this.invalidateChains() + unbuffer() if (this.counter === gen) this.counter = gen - 1 throw err } + // Phase 2 — the at-ack covering sync. Failure here means the fact + // IS in the log (append succeeded) but durability was not promised: + // try to remove it (dropAbove); only a SUCCESSFUL drop earns the + // counter rewind. If the drop itself fails (e.g. the fact was + // sealed by a racing rotation), the generation stays consumed and + // buffered — monotonicity holds, the flush path retries durability, + // and the caller still gets the loud failure. + if (this.logDurability === 'at-ack') { + try { + await this.factLog.ensureSynced() + } catch (err) { + try { + await this.factLog.dropAbove(gen - 1) + unbuffer() + if (this.counter === gen) this.counter = gen - 1 + } catch (dropErr) { + prodLog.warn( + `[GenerationStore] at-ack sync failed for generation ${gen} and the ` + + `appended fact could not be dropped (${(dropErr as Error).message}) — ` + + `the generation stays consumed and buffered; the flush path retries ` + + `durability. Never re-minting a number the log may carry.` + ) + } + throw err + } + } } // Test-only crash simulation. A crash here must cost the buffered // history + the appended fact in 'deferred' mode (open() truncates it diff --git a/tests/integration/sync-fail-compensation.test.ts b/tests/integration/sync-fail-compensation.test.ts new file mode 100644 index 00000000..f5032e34 --- /dev/null +++ b/tests/integration/sync-fail-compensation.test.ts @@ -0,0 +1,78 @@ +/** + * @module tests/integration/sync-fail-compensation + * @description The non-monotonic refusal-loop cure (a production adoption's + * second defect): when the at-ack covering SYNC fails AFTER a successful + * append, the counter must NOT rewind unless the appended fact is provably + * removed — rewinding while the log carries the generation re-mints the + * same number and every later append refuses non-monotonic, wedging the + * write path in a refusal loop ("writes REFUSED until it drains"). + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + vi.restoreAllMocks() + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +describe('at-ack sync-failure compensation', () => { + it('a one-shot sync failure never wedges the write path: the next write mints a FRESH generation and succeeds', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-syncfail-')) + dirs.push(dir) + const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) + await brain.init() // adopt-default: log authority, at-ack + brains.push(brain) + expect(brain.logAuthority().authority).toBe('log') + await brain.add({ data: 'baseline', type: NounType.Document, metadata: { n: 0 } }) + + // Fail exactly ONE covering sync (after its append lands). + // Target ensureSynced (the ACK path's covering sync) — mocking sync() + // itself gets eaten by background flushes before the victim write. + const factLog = (brain as unknown as { + generationStore: { getFactLog(): { ensureSynced(): Promise } } + }).generationStore.getFactLog() + const realEnsure = factLog.ensureSynced.bind(factLog) + let failed = false + vi.spyOn(factLog, 'ensureSynced').mockImplementation(async () => { + if (!failed) { + failed = true + throw new Error('injected sync failure (device hiccup)') + } + return realEnsure() + }) + + // The write whose sync fails: LOUD failure to the caller — never silent. + await expect( + brain.add({ data: 'sync victim', type: NounType.Document, metadata: { n: 1 } }) + ).rejects.toThrow(/sync failure/) + + // THE PIN: the very next write mints a fresh generation and SUCCEEDS — + // no non-monotonic refusal, no refusal loop, regardless of whether the + // failed write's fact was dropped or retained (both are legal outcomes; + // an equal-generation re-mint is not). + const survivor = await brain.add({ data: 'after the storm', type: NounType.Document, metadata: { n: 2 } }) + expect((await brain.get(survivor))!.data).toContain('after the storm') + await brain.flush() + expect(Number.isSafeInteger(brain.generation())).toBe(true) + + // And the log scans clean end-to-end (no torn ordering). + const scan = brain.scanFacts() + let last = 0 + if (scan) { + for await (const batch of (scan as { batches(): AsyncIterable<{ facts: Array<{ generation: number }> }> }).batches()) { + for (const f of batch.facts) { + expect(f.generation, 'strictly ascending').toBeGreaterThan(last) + last = f.generation + } + } + } + expect(last).toBeGreaterThan(0) + }, 120000) +}) diff --git a/tests/unit/db/pad-frame-total.test.ts b/tests/unit/db/pad-frame-total.test.ts new file mode 100644 index 00000000..a5c73d19 --- /dev/null +++ b/tests/unit/db/pad-frame-total.test.ts @@ -0,0 +1,29 @@ +/** + * @module tests/unit/db/pad-frame-total + * @description Pad-frame construction is TOTAL: every size from the minimum + * through 4096+257 is constructible byte-exact (a production adoption found + * the msgpack class-boundary hole at 291 bytes — sync died whole, and the + * failure cascaded into a counter rewind after a successful append). Every + * constructed pad decodes as skip-by-definition filler. + */ +import { describe, it, expect } from 'vitest' +import { encodePadFrame, minPadFrameBytes, decodeGroupV2 } from '../../../src/db/factLogFormat.js' + +describe('pad frames are constructible at EVERY size', () => { + it('exact construction from the minimum through a full sector + boundary spill', () => { + const min = minPadFrameBytes() + for (let size = min; size <= 4096 + 257; size++) { + const frame = encodePadFrame(size) + expect(frame.length, `size ${size}`).toBe(size) + } + }) + + it('the production case (291) and its class-boundary siblings decode as invisible filler', () => { + for (const size of [291, minPadFrameBytes(), 300, 511, 512, 513, 4096]) { + const frame = encodePadFrame(size) + const group = decodeGroupV2(frame) + expect(group.facts, `size ${size} is reader-invisible`).toEqual([]) + expect(group.validBytes).toBe(size) + } + }) +}) From ff43de1ada9f2c48c6d62e2723aebf0e9aeddb30 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 12 Aug 2026 16:56:08 -0700 Subject: [PATCH 003/102] =?UTF-8?q?feat(recovery):=20the=20fold-checkpoint?= =?UTF-8?q?=20bound=20=E2=80=94=20crash=20folds=20(checkpoint,=20head],=20?= =?UTF-8?q?never=20the=20whole=20log=20twice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fold checkpoint (_system/fold-checkpoint.json) is stamped strictly after a canonical-sync barrier over every live entity touched since the last stamp (syncEntityCanonical: ids → canonical paths → fsync; an absent file fsyncs its parent directory so deletes are as durable as writes). An unclean open under log authority now folds only (checkpoint, head]; the chain bootstraps at an empty brain's adoption (three-phase hooks around adoptLogAuthority) or at a brain's first whole-log fold — existing brains converge at their first crash with zero regression. Rollback restores sync immediately; abort paths feed the barrier; a failed barrier retains the old bound (bigger fold later, never a lost write). Five structural pins including boundedness itself. Also: the production-shaped write-flow gate leg (mixed traffic racing flushes, crash mid-traffic, every ack survives — from a consumer-reported gate miss), and two release-ceremony cures (tag-first push so the publish never queues behind the release commit's CI run; raw-curl npmjs shasum probe with propagation grace instead of a one-shot false divergence). --- scripts/release.sh | 39 ++- src/brainy.ts | 20 ++ src/db/generationStore.ts | 257 +++++++++++++++++- src/db/types.ts | 12 + src/storage/adapters/fileSystemStorage.ts | 7 + src/storage/baseStorage.ts | 23 ++ .../integration/fold-checkpoint-bound.test.ts | 200 ++++++++++++++ .../write-flow-production-shape.test.ts | 149 ++++++++++ 8 files changed, 695 insertions(+), 12 deletions(-) create mode 100644 tests/integration/fold-checkpoint-bound.test.ts create mode 100644 tests/integration/write-flow-production-shape.test.ts diff --git a/scripts/release.sh b/scripts/release.sh index ce2d0882..03d60ac2 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -177,8 +177,15 @@ echo -e "${GREEN}✅ Tag created${NC}\n" # Step 9: Push to origin — The Source is the one home (ruled 2026-07-23; the # old public GitHub repo is archived history, no longer part of any release). -echo -e "${BLUE}8️⃣ Pushing to origin...${NC}" -git push --follow-tags origin "$CURRENT_BRANCH" +# TAG FIRST, branch second — deliberately two pushes: the runner is +# sequential, and a combined push can queue the release commit's ci.yml run +# AHEAD of the tag's publish-source run (observed on 10.0.0: the publish sat +# ~37 minutes behind a redundant CI run of the very commit the local gates +# had just proven). Pushing the tag alone queues the publish immediately; +# the branch push (and its ci.yml run) follows behind it, harmlessly. +echo -e "${BLUE}8️⃣ Pushing to origin (tag first — the publish must never queue behind CI)...${NC}" +git push origin "v${NEW_VERSION}" +git push origin "$CURRENT_BRANCH" echo -e "${GREEN}✅ Pushed to origin${NC}\n" # Step 10: The home publish (The Source, source.soulcraft.com) is CI's job @@ -227,14 +234,32 @@ npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://re rm -rf "$STOREFRONT_TMP" # Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true -# Verify the pair is byte-identical by registry-reported shasum — divergence here -# means the storefront leg must be treated as failed, loudly. +# Verify the pair is byte-identical by registry-reported shasum — divergence +# here means the storefront leg must be treated as failed, loudly. RETRIED +# with raw curl: npmjs metadata propagates with a lag measured in minutes, +# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a +# raw curl of the registry document already confirmed byte-identity. The +# probe now reads the registry JSON directly (no npm cache in the path) and +# gives propagation up to 5 minutes before calling the pair divergent. +NPMJS_VERIFY_ATTEMPTS=20 +NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable") -NPMJS_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=https://registry.npmjs.org/" 2>/dev/null || echo "npmjs-unavailable") -if [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then +PAIR_IDENTICAL=false +for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do + NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \ + | node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \ + || echo "") + if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then + PAIR_IDENTICAL=true + break + fi + echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}" + sleep "$NPMJS_VERIFY_INTERVAL_S" +done +if [ "$PAIR_IDENTICAL" = true ]; then echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n" else - echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} — investigate before announcing${NC}\n" + echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n" exit 1 fi diff --git a/src/brainy.ts b/src/brainy.ts index 785d10e5..dc97b82f 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8239,6 +8239,23 @@ export class Brainy implements BrainyInterface { async adoptLogAuthority(): Promise { await this.ensureInitialized() this.assertWritable('adoptLogAuthority') + // Fold-checkpoint chain, phase 1: a FRESH brain (no committed + // generations) arms the chain now so the backfill's re-commits below + // feed the canonical-sync accumulator — its first stamp is then total. + // A non-fresh flip skips (the store refuses the arm); its chain starts + // at the first recovery fold instead. Disarmed on any failure below. + this.generationStore.beginFoldCheckpointBootstrap() + try { + return await this.adoptLogAuthorityInner() + } catch (err) { + this.generationStore.abandonFoldCheckpointBootstrap() + throw err + } + } + + /** The adoption body — see {@link Brainy.adoptLogAuthority} (which owns the + * fold-checkpoint bootstrap arm/disarm around it). */ + private async adoptLogAuthorityInner(): Promise { let report = await this.verifyLogAuthority() // BASELINE BACKFILL: curable divergences are rows whose CANONICAL truth @@ -8334,6 +8351,9 @@ export class Brainy implements BrainyInterface { report ) this.generationStore.setLogDurability('at-ack') + // Fold-checkpoint chain, phase 2: the flip is recorded — open the stamp + // gate so the next flush/close barrier writes the first checkpoint. + this.generationStore.completeFoldCheckpointBootstrap() return report } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 8f3bf625..837ea90a 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -78,10 +78,21 @@ export const MANIFEST_PATH = '_system/manifest.json' /** * The clean-shutdown marker (log-authority recovery gate): written+fsynced at * a clean close carrying the committed generation; CONSUMED at every open. - * Absent or generation-mismatched at open = unclean shutdown = the whole-log - * replay fold. Its absence is always safe (costs one replay, loses nothing). + * Absent or generation-mismatched at open = unclean shutdown = the replay + * fold, bounded below by the fold checkpoint when one is stored (whole-log + * without one). Its absence is always safe (costs one fold, loses nothing). */ export const CLEAN_SHUTDOWN_PATH = '_system/clean-shutdown.json' +/** + * The fold checkpoint (log-authority recovery BOUND): `{ generation: G }` + * asserts that every entity whose latest fact is ≤ G has durable canonical + * bytes — so an unclean open folds only `(G, head]` instead of the whole log. + * Stamped strictly AFTER a canonical-sync barrier over every live entity + * touched since the last stamp (stamp-after-data); absent or torn = fold from + * 0 (always safe, just bigger). The chain of stamps starts only at a provable + * point: an empty brain, or the end of a whole-log fold. + */ +export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' @@ -219,6 +230,37 @@ export class GenerationStore { /** Compaction horizon — record-sets ≤ this are reclaimed. */ private horizonGen = 0 + /** + * Fold-checkpoint accumulator: every entity whose CANONICAL live bytes were + * (re)written since the last stamped checkpoint. Drained by + * {@link advanceFoldCheckpointUnlocked} — synced first, stamped after; on a + * failed barrier the drained ids merge back so the checkpoint can never + * advance past unsynced bytes. Fed only while the chain is valid (see + * {@link foldCheckpointChainValid}) so tree-authority brains never grow it. + */ + private checkpointDirtyNouns = new Set() + /** @see checkpointDirtyNouns — the verb half of the accumulator. */ + private checkpointDirtyVerbs = new Set() + /** + * Whether the checkpoint chain is PROVABLY sound for this brain: true when + * a stored checkpoint exists (induction), the brain opened empty (vacuous), + * or a whole-log fold just re-applied every fact (base case). While false, + * checkpoints are never stamped and the fold bound stays 0 — the honest + * 10.0 contract, upgraded at the brain's first recovery fold. + */ + private foldCheckpointChainValid = false + /** Last stamped fold-checkpoint generation (0 = none / fold from origin). */ + private foldCheckpoint = 0 + /** + * Whether this brain's stored authority is the log — set from the stored + * artifact at open, or by {@link completeFoldCheckpointBootstrap} when an + * in-session adoption flips it. Checkpoints are only ever STAMPED under log + * authority (the artifact bounds the log fold, which only log-authority + * recovery runs); the dirty accumulator may fill slightly earlier, during + * an adoption in flight (see {@link beginFoldCheckpointBootstrap}). + */ + private authorityIsLog = false + /** * Committed generations whose record dirs exist, stored as a SORTED, DISJOINT, * ascending list of INCLUSIVE `[start, end]` intervals (a run-length set). @@ -552,6 +594,7 @@ export class GenerationStore { // drift machinery at open — same as group-commit recovery. const authority = await readLogAuthority(this.storage) if (authority.authority === 'log') { + this.authorityIsLog = true // TWO REPLAY TIERS, gated by the clean-shutdown marker: // // (1) ABOVE-MANIFEST (always): an intact fact above the manifest is @@ -574,8 +617,22 @@ export class GenerationStore { const cleanShutdown = await this.readCleanShutdownMarker() const orphans = await this.factLog.peekFactsAbove(this.committed) const uncleanOpen = cleanShutdown === null || cleanShutdown !== this.committed + // FOLD-CHECKPOINT BOUND: a stored checkpoint G proves every entity + // whose latest fact is ≤ G has durable canonical bytes (each stamp + // followed a canonical-sync barrier), so the unclean fold only needs + // (G, head] — entities untouched since G are already safe, entities + // touched after G get their latest after-image re-applied. Absent or + // invalid checkpoint = fold from 0 (the 10.0 whole-log contract). + const checkpoint = await this.readFoldCheckpoint() + const foldBound = checkpoint ?? 0 + // Chain validity: induction (a stored stamp), vacuous truth (an empty + // brain has no bytes to assert), or — set below — the base case (a + // whole-log fold re-applies and re-syncs every entity in the log). + this.foldCheckpointChainValid = checkpoint !== null || this.committed === 0 + this.foldCheckpoint = foldBound + if (uncleanOpen) this.foldCheckpointChainValid = true const factsToReplay = uncleanOpen - ? await this.factLog.peekFactsAbove(0) + ? await this.factLog.peekFactsAbove(foldBound) : orphans if (factsToReplay.length > 0) { let replayed = 0 @@ -587,6 +644,7 @@ export class GenerationStore { : { metadata: op.record.metadata, vector: op.record.vector } if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) else await this.storage.writeNounRaw(op.id, image) + this.noteCheckpointDirty(op.kind, op.id) } replayed++ if (fact.generation > this.committed) { @@ -612,10 +670,21 @@ export class GenerationStore { await this.storage.syncRawObjects([MANIFEST_PATH]) prodLog.warn( `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + - `canonical (${uncleanOpen ? 'WHOLE-LOG fold — unclean shutdown' : 'above-manifest'}; ` + - `committed at ${this.committed}) — an acked write is never lost` + `canonical (${ + uncleanOpen + ? foldBound > 0 + ? `BOUNDED fold above checkpoint ${foldBound} — unclean shutdown` + : 'WHOLE-LOG fold — unclean shutdown' + : 'above-manifest' + }; committed at ${this.committed}) — an acked write is never lost` ) } + // A recovery fold re-applied (and the barrier below re-syncs) every + // entity in (bound, head] — stamp the checkpoint at the new committed + // watermark so the NEXT crash folds only its own tail. This is also + // the chain's base case: the first whole-log fold of a pre-checkpoint + // brain covers every entity in the log, so its stamp is total. + if (uncleanOpen) await this.advanceFoldCheckpointUnlocked() // The marker is consumed: any session that can write invalidates it // at first commit (see the commit paths); a clean close re-writes it. await this.clearCleanShutdownMarker() @@ -672,6 +741,12 @@ export class GenerationStore { await this.flushPendingSingleOps() this.storage.setGenerationBumpHook(undefined) await this.persistCounterNow() + // Fold-checkpoint barrier BEFORE the clean-shutdown marker: entities that + // reached the accumulator outside the pending tier (transact commits, + // aborted-write restores) get their canonical bytes synced and the stamp + // advanced, so the marker below never vouches for bytes the checkpoint + // chain hasn't proven durable. + await this.advanceFoldCheckpoint() // Clean-shutdown marker (log-authority recovery gate): everything above // is durable; stamp the committed generation so the next open can adopt // instead of folding the log. Written LAST — a crash before this line is @@ -705,6 +780,131 @@ export class GenerationStore { } } + /** + * Read the fold checkpoint's generation, or `null` when absent, torn, or + * implausible (> committed) — every invalid shape degrades to the safe + * whole-log fold, never to a bound that could skip an acked write. + */ + private async readFoldCheckpoint(): Promise { + try { + const raw = (await this.storage.readRawObject(FOLD_CHECKPOINT_PATH)) as { + generation?: number + } | null + const gen = raw?.generation + if (!Number.isSafeInteger(gen) || (gen as number) < 0) return null + if ((gen as number) > this.committed) { + prodLog.warn( + `[GenerationStore] fold checkpoint ${gen} is ahead of the manifest ` + + `(${this.committed}) — ignoring it; recovery folds the whole log` + ) + return null + } + return gen as number + } catch { + return null + } + } + + /** + * Record that an entity's canonical live bytes were (re)written and are not + * yet covered by a checkpoint stamp. Gated on chain validity so brains + * without a sound chain (tree authority, or log authority before its first + * recovery fold) never accumulate — they keep the fold-from-0 contract. + */ + private noteCheckpointDirty(kind: 'noun' | 'verb', id: string): void { + if (!this.foldCheckpointChainValid) return + if (kind === 'verb') this.checkpointDirtyVerbs.add(id) + else this.checkpointDirtyNouns.add(id) + } + + /** + * The canonical-sync barrier + checkpoint stamp (must run under the commit + * mutex or in single-threaded open). Drains the dirty accumulator, makes + * those entities' canonical bytes durable via the adapter barrier, and only + * THEN stamps `_system/fold-checkpoint.json` at the committed watermark — + * stamp-after-data, always. On any failure the drained ids merge back and + * the stored checkpoint stays where it was: the bound can lag (a bigger + * fold later) but can never overstate durability (a lost write, outlawed). + */ + private async advanceFoldCheckpointUnlocked(): Promise { + if (!this.foldCheckpointChainValid || !this.authorityIsLog || !this.factLog) return + const nouns = [...this.checkpointDirtyNouns] + const verbs = [...this.checkpointDirtyVerbs] + const target = this.committed + if (nouns.length === 0 && verbs.length === 0 && target === this.foldCheckpoint) return + this.checkpointDirtyNouns = new Set() + this.checkpointDirtyVerbs = new Set() + try { + if (nouns.length > 0 || verbs.length > 0) { + await this.storage.syncEntityCanonical?.(nouns, verbs) + } + await this.storage.writeRawObject(FOLD_CHECKPOINT_PATH, { generation: target }) + await this.storage.syncRawObjects([FOLD_CHECKPOINT_PATH]) + this.foldCheckpoint = target + } catch (err) { + for (const id of nouns) this.checkpointDirtyNouns.add(id) + for (const id of verbs) this.checkpointDirtyVerbs.add(id) + prodLog.warn( + `[GenerationStore] fold-checkpoint barrier failed at generation ${target} ` + + `(${(err as Error).message}) — checkpoint stays at ${this.foldCheckpoint}; ` + + `recovery would fold from there (bigger, never lossy). Will retry next flush.` + ) + } + } + + /** + * @description Public, mutex-serialized fold-checkpoint advance — called by + * `close()` after the final flush so entities touched by paths that do not + * ride the pending tier (e.g. `transact()`) are covered before the + * clean-shutdown marker is written. + */ + async advanceFoldCheckpoint(): Promise { + return this.withMutex(() => this.advanceFoldCheckpointUnlocked()) + } + + /** + * @description Adoption-time chain bootstrap, phase 1 — called by + * `adoptLogAuthority()` BEFORE its oracle/backfill passes. Only a FRESH + * brain (committed === 0) may bootstrap here: with no committed + * generations the chain's assertion is vacuously true, and arming it now + * means the baseline backfill's own re-commits feed the dirty accumulator, + * so the first stamp after the flip covers them. A non-fresh flip skips + * this (returns false) — its chain starts at the brain's first recovery + * fold instead, because only a whole-log fold can prove coverage of + * entities written before the log existed. + */ + beginFoldCheckpointBootstrap(): boolean { + if (this.committed !== 0 || this.foldCheckpointChainValid) { + return this.foldCheckpointChainValid + } + this.foldCheckpointChainValid = true + this.foldCheckpoint = 0 + return true + } + + /** + * @description Adoption-time chain bootstrap, phase 2 — called after + * `flipToLogAuthority` records the flip. Opens the stamp gate; the next + * flush/close barrier writes the first checkpoint. + */ + completeFoldCheckpointBootstrap(): void { + this.authorityIsLog = true + } + + /** + * @description Adoption-time chain bootstrap, abort — called when an + * adoption attempt throws or refuses after phase 1. Disarms the chain and + * drops the accumulator so a tree-authority brain never accumulates or + * stamps. (If the chain was valid BEFORE the attempt — a stored checkpoint + * exists — it stays valid; only a phase-1 arm is undone.) + */ + abandonFoldCheckpointBootstrap(): void { + if (this.authorityIsLog) return + this.foldCheckpointChainValid = false + this.checkpointDirtyNouns = new Set() + this.checkpointDirtyVerbs = new Set() + } + /** * @description TEST-ONLY: install (or clear, with `undefined`) a fault * injector that is invoked at each {@link CommitFaultPhase} of the commit @@ -1238,6 +1438,13 @@ export class GenerationStore { this.historyBytesTotal += delta.bytes ?? 0 } this.extendChains(gen, nouns, verbs) + // Fold-checkpoint accounting: the write barrier above already synced + // this batch's canonical footprint on adapters that have one, but the + // accumulator entry is the belt — an adapter without a write barrier + // still gets these ids covered by the next checkpoint barrier, and a + // redundant fsync of already-durable bytes is cheap and idempotent. + for (const id of nouns) this.noteCheckpointDirty('noun', id) + for (const id of verbs) this.noteCheckpointDirty('verb', id) const logEntry: TxLogEntry = { generation: gen, timestamp, ...(args.meta && { meta: args.meta }) } await this.storage.appendTxLogLine(JSON.stringify(logEntry)) @@ -1249,6 +1456,13 @@ export class GenerationStore { if (crashSimulated) { throw err } + // Fold-checkpoint accounting: an abort's rollback restores are raw + // canonical writes that never reach the transaction write barrier + // (flushWriteBarrier only runs on the commit path) — feed them so the + // next checkpoint barrier syncs the restored bytes before any stamp + // vouches for them. + for (const id of nouns) this.noteCheckpointDirty('noun', id) + for (const id of verbs) this.noteCheckpointDirty('verb', id) // The trapdoor for a batch: if rollback FAILED to fully apply, canonical // storage may be inconsistent. A batch is never adopted forward (its // other ops were rolled back — partial commit would break atomicity), so @@ -1476,6 +1690,13 @@ export class GenerationStore { await args.execute() } catch (err) { this.inTransact = false + // Fold-checkpoint accounting: execute() ran, so canonical bytes for + // the touched ids changed — whether they now hold the new images, a + // restored rollback, or (the trapdoor) something indeterminate, the + // next checkpoint stamp must not assert their durability without a + // barrier over whatever is actually there. + for (const id of nouns) this.noteCheckpointDirty('noun', id) + for (const id of verbs) this.noteCheckpointDirty('verb', id) // A failed rollback (TransactionRollbackError) may have left canonical // storage inconsistent — the trapdoor. Reconcile against the // before-images to decide the honest response (David's ruling: @@ -1530,6 +1751,10 @@ export class GenerationStore { throw err } this.inTransact = false + // Fold-checkpoint accounting: the live canonical write is applied — it + // must ride the next canonical-sync barrier before any stamp covers it. + for (const id of nouns) this.noteCheckpointDirty('noun', id) + for (const id of verbs) this.noteCheckpointDirty('verb', id) // Test-only crash simulation (direct call — a throw propagates with no // cleanup, exactly like a process death; recovery-on-open restores the // contract). A crash here must cost only the never-returned ack: the @@ -1801,6 +2026,16 @@ export class GenerationStore { for (const entry of logEntries) { await this.storage.appendTxLogLine(JSON.stringify(entry)) } + + // Fold-checkpoint barrier: the window's LIVE canonical bytes (the acked + // writes themselves — the staging sync above covered only their history + // copies) become durable here, and only then does the checkpoint stamp + // advance to the new committed watermark. This is what keeps crash + // recovery's log fold bounded to (checkpoint, head] instead of the + // whole log. A failure inside is absorbed by the barrier (it warns, + // retains the accumulator, and leaves the old bound standing) — history + // durability above already succeeded, so the flush itself is good. + await this.advanceFoldCheckpointUnlocked() }) } @@ -2961,6 +3196,8 @@ export class GenerationStore { private async rollBackUncommittedGeneration(gen: number): Promise { const dir = `${GENERATIONS_PREFIX}/${gen}` const prevPaths = await this.storage.listRawObjects(`${dir}/prev`) + const restoredNouns: string[] = [] + const restoredVerbs: string[] = [] for (const recordPath of prevPaths) { const id = recordIdFromPath(recordPath) if (id === null) continue @@ -2969,10 +3206,20 @@ export class GenerationStore { const image = { metadata: record.metadata, vector: record.vector } if (record.kind === 'verb') { await this.storage.writeVerbRaw(id, image) + restoredVerbs.push(id) } else { await this.storage.writeNounRaw(id, image) + restoredNouns.push(id) } } + // Make the restores durable IMMEDIATELY (this runs at open, before the + // fold-checkpoint chain state is even read): a restored before-image + // replaces bytes a stored checkpoint may already vouch for, so it must + // reach disk with the same certainty — otherwise a power cut could let + // the rolled-back write's bytes resurrect past a bounded fold. + if (restoredNouns.length > 0 || restoredVerbs.length > 0) { + await this.storage.syncEntityCanonical?.(restoredNouns, restoredVerbs) + } await this.storage.removeRawPrefix(dir) prodLog.warn( `[GenerationStore] rolled back uncommitted generation ${gen} ` + diff --git a/src/db/types.ts b/src/db/types.ts index a7f86a89..2c7eab8f 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -462,6 +462,18 @@ export interface GenerationStorage { /** @see beginWriteBarrier — fsync every canonical write since begin. */ flushWriteBarrier?(): Promise + /** + * OPTIONAL fold-checkpoint durability barrier: make the listed entities' + * CANONICAL live objects durable — fsync each present metadata/vector file + * AND the parent directory entry of each absent one (so a delete is as + * durable as a write). The generation store may only advance the fold + * checkpoint (`_system/fold-checkpoint.json`) after this resolves; the + * checkpoint bounds crash recovery's log fold to `(checkpoint, head]`. + * Adapters whose writes are durable per-call may leave this undefined — + * the store then treats canonical durability as immediate. + */ + syncEntityCanonical?(nouns: string[], verbs: string[]): Promise + /** Read an entity's raw stored metadata+vector objects. */ readNounRaw(id: string): Promise<{ metadata: any | null; vector: any | null }> /** Restore an entity's raw stored objects (`null` part ⇒ delete that file). */ diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index fea817d5..81c6e545 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -799,6 +799,7 @@ export class FileSystemStorage extends BaseStorage { for (const objectPath of paths) { const fullPath = path.join(this.rootDir, objectPath) + let synced = false for (const candidate of [`${fullPath}.gz`, fullPath]) { let handle: any try { @@ -813,8 +814,14 @@ export class FileSystemStorage extends BaseStorage { await handle.close() } parentDirs.add(path.dirname(fullPath)) + synced = true break } + // An absent path is a state too: fsync the parent directory so a + // completed unlink is durable (a delete must survive power loss as + // surely as a write — otherwise a bounded log fold could let a + // tombstoned record resurrect from a lost directory update). + if (!synced) parentDirs.add(path.dirname(fullPath)) } for (const dir of parentDirs) { diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 23003ede..c3b3b3bf 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1390,6 +1390,29 @@ export abstract class BaseStorage extends BaseStorageAdapter { void paths } + /** + * Fold-checkpoint durability barrier: make the listed entities' canonical + * live objects durable. Maps each id to its canonical metadata + vector + * paths and delegates to {@link BaseStorage.syncRawObjects}, whose + * filesystem override fsyncs present files (and their rename directory + * entries) and the parent directory of absent ones — so deletes are as + * durable as writes. The generation store advances the fold checkpoint + * only after this resolves (stamp-after-data). + * + * @param nouns - Entity ids whose canonical objects must be durable. + * @param verbs - Relationship ids whose canonical objects must be durable. + */ + public async syncEntityCanonical(nouns: string[], verbs: string[]): Promise { + const paths: string[] = [] + for (const id of nouns) { + paths.push(getNounMetadataPath(id), getNounVectorPath(id)) + } + for (const id of verbs) { + paths.push(getVerbMetadataPath(id), getVerbVectorPath(id)) + } + if (paths.length > 0) await this.syncRawObjects(paths) + } + /** * Read an entity's raw stored objects — the exact bytes at its canonical * metadata + vector paths (write-cache coherent). Used by the generation diff --git a/tests/integration/fold-checkpoint-bound.test.ts b/tests/integration/fold-checkpoint-bound.test.ts new file mode 100644 index 00000000..ce074dcf --- /dev/null +++ b/tests/integration/fold-checkpoint-bound.test.ts @@ -0,0 +1,200 @@ +/** + * @module tests/integration/fold-checkpoint-bound + * @description The fold-checkpoint bound (crash recovery's log fold, bounded): + * `_system/fold-checkpoint.json` at generation G asserts every entity whose + * latest fact is ≤ G has DURABLE canonical bytes — each stamp strictly follows + * a canonical-sync barrier over every live entity touched since the last one + * (stamp-after-data). An unclean open then folds only `(G, head]` instead of + * the whole log. These pins prove the four load-bearing properties: + * + * 1. The stamp exists and tracks the committed watermark (flush + close). + * 2. The fold is genuinely BOUNDED — facts ≤ G are skipped — while facts in + * `(G, head]` are re-applied even BELOW the manifest. + * 3. A failed barrier NEVER advances the stamp (the bound can lag, growing + * a later fold — it can never overstate durability, losing a write). + * 4. A pre-checkpoint brain (the 10.0 shape) bootstraps its chain at its + * first whole-log fold; a tree-authority brain never stamps at all. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as zlib from 'node:zlib' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { + abandonAsCrashed, + dropCanonicalNoun, + makeTempDir, + openBrain, + storeOf +} from '../helpers/durabilityKillMatrix.js' + +const CHECKPOINT = join('_system', 'fold-checkpoint.json') + +/** Read the fold-checkpoint artifact's generation from disk, or null. */ +function readCheckpoint(dir: string): number | null { + for (const candidate of [join(dir, `${CHECKPOINT}.gz`), join(dir, CHECKPOINT)]) { + if (!fs.existsSync(candidate)) continue + const raw = fs.readFileSync(candidate) + const text = candidate.endsWith('.gz') ? zlib.gunzipSync(raw).toString('utf8') : raw.toString('utf8') + const parsed = JSON.parse(text) as { generation?: number } + return Number.isSafeInteger(parsed.generation) ? (parsed.generation as number) : null + } + return null +} + +function removeArtifact(dir: string, rel: string): void { + for (const candidate of [join(dir, `${rel}.gz`), join(dir, rel)]) { + fs.rmSync(candidate, { force: true }) + } +} + +function committedOf(brain: Brainy): number { + return (storeOf(brain) as unknown as { committed: number }).committed +} + +describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], never less durability than stamped', () => { + const dirs: string[] = [] + const liveBrains: Brainy[] = [] + afterEach(async () => { + vi.restoreAllMocks() + for (const b of liveBrains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) + }) + function trackDir(): string { + const dir = makeTempDir() + dirs.push(dir) + return dir + } + + it('a fresh adopt brain stamps at flush and again at close — the stamp tracks the committed watermark', async () => { + const dir = trackDir() + const brain = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(brain) + expect(brain.logAuthority().authority).toBe('log') + + await brain.add({ data: 'first', type: NounType.Document, metadata: { n: 1 } }) + await brain.add({ data: 'second', type: NounType.Document, metadata: { n: 2 } }) + await brain.flush() + const afterFlush = readCheckpoint(dir) + expect(afterFlush).toBe(committedOf(brain)) + expect(afterFlush!).toBeGreaterThan(0) + + await brain.add({ data: 'third', type: NounType.Document, metadata: { n: 3 } }) + const closingCommit = liveBrains.pop()! + await closingCommit.close() + // Close flushes, so the stamp advanced with it — and the clean-shutdown + // marker it writes afterward never vouches for bytes the stamp has not. + expect(readCheckpoint(dir)).toBeGreaterThanOrEqual(afterFlush!) + }, 120000) + + it('BOUNDED fold: facts ≤ checkpoint are skipped, facts in (checkpoint, head] are re-applied even below the manifest; a failed barrier retains the old bound', async () => { + const dir = trackDir() + const brain = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(brain) + + // Window 1 — flushed and stamped: the checkpoint's covered past. + const idA = await brain.add({ data: 'covered by the stamp', type: NounType.Document, metadata: { w: 1 } }) + await brain.flush() + const checkpoint1 = readCheckpoint(dir) + expect(checkpoint1).toBe(committedOf(brain)) + + // Window 2 — committed BELOW a new manifest but with the checkpoint stamp + // FAILING: the barrier throws once, so the manifest advances while the + // stamp stays at checkpoint1 (pin 3: a failed barrier never advances it). + const storage = (brain as unknown as { + storage: { syncEntityCanonical(n: string[], v: string[]): Promise } + }).storage + const realBarrier = storage.syncEntityCanonical.bind(storage) + let failedOnce = false + vi.spyOn(storage, 'syncEntityCanonical').mockImplementation(async (n: string[], v: string[]) => { + if (!failedOnce) { + failedOnce = true + throw new Error('injected barrier failure (device hiccup)') + } + return realBarrier(n, v) + }) + const idB = await brain.add({ data: 'below manifest, above checkpoint', type: NounType.Document, metadata: { w: 2 } }) + await brain.flush() + expect(failedOnce).toBe(true) + expect(readCheckpoint(dir)).toBe(checkpoint1) // stamp did NOT advance + expect(committedOf(brain)).toBeGreaterThan(checkpoint1!) // manifest DID + + // Crash. Vaporize BOTH canonical records: idB's fact lives in + // (checkpoint, manifest] — the bounded fold MUST restore it; idA's fact + // is ≤ checkpoint — the fold must SKIP it (its loss here is synthetic: + // the stamp's barrier fsynced it, a power cut cannot take it, and the + // skip is exactly what makes the fold bounded instead of whole-log). + await abandonAsCrashed(liveBrains.pop()!) + dropCanonicalNoun(dir, idA) + dropCanonicalNoun(dir, idB) + + const reopened = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(reopened) + const restoredB = await reopened.get(idB) + expect(restoredB, 'a fact above the checkpoint is re-applied even below the manifest').not.toBeNull() + const skippedA = await reopened.get(idA) + expect(skippedA, 'a fact at-or-below the checkpoint is outside the fold — the bound is real').toBeNull() + // And recovery re-stamped at its new committed watermark. + expect(readCheckpoint(dir)).toBe(committedOf(reopened)) + }, 120000) + + it('a pre-checkpoint brain (the 10.0 shape) folds the WHOLE log once, then its chain is established', async () => { + const dir = trackDir() + const brain = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(brain) + const idA = await brain.add({ data: 'ten-point-oh resident', type: NounType.Document, metadata: { era: '10.0' } }) + await brain.flush() + await liveBrains.pop()!.close() + + // Rewind the brain to the 10.0 shape: no checkpoint artifact, and an + // unclean shutdown (marker gone) — exactly what an existing fleet brain + // looks like at its first crash under 10.1. + removeArtifact(dir, CHECKPOINT) + removeArtifact(dir, join('_system', 'clean-shutdown.json')) + dropCanonicalNoun(dir, idA) + + const reopened = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(reopened) + expect(await reopened.get(idA), 'no checkpoint ⇒ whole-log fold ⇒ every acked write restored').not.toBeNull() + const stamped = readCheckpoint(dir) + expect(stamped, 'the first whole-log fold is the chain’s base case — it stamps').toBe(committedOf(reopened)) + }, 120000) + + it('a tree-authority brain never stamps a checkpoint', async () => { + const dir = trackDir() + const brain = await openBrain(dir, { logAuthority: 'defer' }) + liveBrains.push(brain) + expect(brain.logAuthority().authority).not.toBe('log') + await brain.add({ data: 'tree resident', type: NounType.Document, metadata: { n: 1 } }) + await brain.flush() + await liveBrains.pop()!.close() + expect(readCheckpoint(dir)).toBeNull() + }, 120000) + + it('a delete rides the barrier: the tombstoned id is in the synced set and the stamp advances past it', async () => { + const dir = trackDir() + const brain = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(brain) + const id = await brain.add({ data: 'short-lived', type: NounType.Document, metadata: { n: 1 } }) + await brain.flush() + + const storage = (brain as unknown as { + storage: { syncEntityCanonical(n: string[], v: string[]): Promise } + }).storage + const seen: string[][] = [] + const realBarrier = storage.syncEntityCanonical.bind(storage) + vi.spyOn(storage, 'syncEntityCanonical').mockImplementation(async (n: string[], v: string[]) => { + seen.push([...n]) + return realBarrier(n, v) + }) + + await brain.remove(id) + await brain.flush() + expect( + seen.some((nouns) => nouns.includes(id)), + 'the deleted id must reach the canonical barrier (absence is durable state too)' + ).toBe(true) + expect(readCheckpoint(dir)).toBe(committedOf(brain)) + }, 120000) +}) diff --git a/tests/integration/write-flow-production-shape.test.ts b/tests/integration/write-flow-production-shape.test.ts new file mode 100644 index 00000000..f33cdc88 --- /dev/null +++ b/tests/integration/write-flow-production-shape.test.ts @@ -0,0 +1,149 @@ +/** + * @module tests/integration/write-flow-production-shape + * @description The production-shaped WRITE-FLOW gate leg. A downstream + * deployment's release gate went all-green on snapshots and rehearsal reads + * while two write-path defects (pad-frame constructibility, a counter rewind + * after a successful append) waited in ordinary WRITE flows — deferred + * embedding retries plus background history-flush concurrency wearing the + * stacks. This leg runs that exact shape, permanently: + * + * - concurrent mixed writes (adds, deferred-embed adds, updates, removes) + * - racing explicit flushes (the history tier's group commit, mid-traffic) + * - then the three laws: every ack is readable truth, the fact log is + * STRICTLY ascending end-to-end, and no write is ever refused. + * + * Part two crashes the brain mid-traffic (no close — RAM discarded) and + * requires every acked write back after reopen: the at-ack contract under + * the same production shape, not under a synthetic single write. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { + abandonAsCrashed, + factGenerations, + makeTempDir, + openBrain +} from '../helpers/durabilityKillMatrix.js' + +describe('write-flow production shape — the pair gate leg from a consumer-reported miss', () => { + const dirs: string[] = [] + const liveBrains: Brainy[] = [] + afterEach(async () => { + for (const b of liveBrains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) + }) + function trackDir(): string { + const dir = makeTempDir() + dirs.push(dir) + return dir + } + + async function runTrafficWave( + brain: Brainy, + wave: number, + perWave: number + ): Promise<{ kept: string[]; removed: string[] }> { + const kept: string[] = [] + const removed: string[] = [] + const work: Promise[] = [] + for (let i = 0; i < perWave; i++) { + const n = wave * perWave + i + if (i % 4 === 0) { + // Deferred-embed add — the retry-marker flow that wore the defect. + work.push( + brain + .add({ data: `deferred payload ${n}`, type: NounType.Document, metadata: { n, defer: true }, deferEmbedding: true }) + .then((id) => void kept.push(id)) + ) + } else if (i % 4 === 1) { + // Add, then update it in the same wave (two generations, same id). + work.push( + brain.add({ data: `versioned payload ${n}`, type: NounType.Document, metadata: { n, v: 1 } }).then(async (id) => { + kept.push(id) + await brain.update({ id, metadata: { n, v: 2 } }) + }) + ) + } else if (i % 4 === 2) { + // Add, then remove — a durable tombstone is an ack too. + work.push( + brain.add({ data: `ephemeral payload ${n}`, type: NounType.Document, metadata: { n } }).then(async (id) => { + await brain.remove(id) + removed.push(id) + }) + ) + } else { + work.push( + brain.add({ data: `plain payload ${n}`, type: NounType.Document, metadata: { n } }).then((id) => void kept.push(id)) + ) + } + // Race the history tier's group commit against live traffic. + if (i % 5 === 3) work.push(brain.flush()) + } + // NO REFUSALS: every promise must resolve — a single rejection here is + // the refusal-loop costume this leg exists to catch. + await Promise.all(work) + return { kept, removed } + } + + it('three waves of mixed traffic with racing flushes: every ack is truth, the log is strictly ascending, nothing refused', async () => { + const dir = trackDir() + const brain = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(brain) + expect(brain.logAuthority().authority).toBe('log') + + const kept: string[] = [] + const removed: string[] = [] + for (let wave = 0; wave < 3; wave++) { + const result = await runTrafficWave(brain, wave, 20) + kept.push(...result.kept) + removed.push(...result.removed) + } + await brain.flush() + + for (const id of kept) { + expect(await brain.get(id), `acked write ${id} must be readable truth`).not.toBeNull() + } + for (const id of removed) { + expect(await brain.get(id), `acked remove ${id} must hold`).toBeNull() + } + + const gens = await factGenerations(brain) + expect(gens.length).toBeGreaterThan(0) + for (let i = 1; i < gens.length; i++) { + expect(gens[i], 'fact log strictly ascending end-to-end').toBeGreaterThan(gens[i - 1]) + } + + // Clean reopen: the same truth survives a restart. + await liveBrains.pop()!.close() + const reopened = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(reopened) + for (const id of kept.slice(0, 10)) { + expect(await reopened.get(id)).not.toBeNull() + } + }, 240000) + + it('crash mid-traffic: every acked write survives the reopen (the at-ack law under the production shape)', async () => { + const dir = trackDir() + const brain = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(brain) + + const { kept, removed } = await runTrafficWave(brain, 0, 24) + // No close, no flush — the process "dies" holding its RAM. + await abandonAsCrashed(liveBrains.pop()!) + + const reopened = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(reopened) + for (const id of kept) { + expect(await reopened.get(id), `acked write ${id} must survive the crash`).not.toBeNull() + } + for (const id of removed) { + expect(await reopened.get(id), `acked remove ${id} must survive the crash`).toBeNull() + } + const gens = await factGenerations(reopened) + for (let i = 1; i < gens.length; i++) { + expect(gens[i], 'fact log strictly ascending after recovery').toBeGreaterThan(gens[i - 1]) + } + }, 240000) +}) From 9ca80667c379f661d56db38df17f6d3cdb3510e0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 13 Aug 2026 09:19:14 -0700 Subject: [PATCH 004/102] =?UTF-8?q?fix(restore):=20a=20restore=20is=20an?= =?UTF-8?q?=20unclean=20event=20=E2=80=94=20the=20swap=20runs=20quiesced?= =?UTF-8?q?=20and=20the=20snapshot's=20durability=20stamps=20never=20survi?= =?UTF-8?q?ve=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects with one root, found by the fold-checkpoint work's first integration gate. (1) THE RACE: restore() never quiesced the generation store, so a background flush could write into _system/ while the swap was removing it — observed as ENOTEMPTY mid-swap when a checkpoint stamp landed between readdir and rmdir. The swap now runs inside the store's exclusive section (runStateReplacement): flush timer disarmed, pending tier and checkpoint accumulator discarded BEFORE any directory moves. (2) THE INHERITED ASSERTION: a snapshot carries its source brain's clean-shutdown marker and fold checkpoint, but the restored files were bulk-copied without per-file fsync — the inherited stamps would suppress exactly the recovery fold that cures a post-restore power cut. reopenAfterRestore now deletes both stamps before reopening: the open treats the store as uncleanly shut, folds the restored log into canonical, barrier-syncs what it re-applied, and stamps fresh — the restored state is durably founded at restore time instead of borrowing assertions about bytes this disk never synced. Pinned: restore under in-flight traffic completes; the pre-restore stamp does not survive; the post-restore stamp is the reopen fold's own, at the restored watermark. --- src/brainy.ts | 8 +++- src/db/generationStore.ts | 43 +++++++++++++++++++ .../integration/fold-checkpoint-bound.test.ts | 35 +++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/brainy.ts b/src/brainy.ts index dc97b82f..d7313855 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -9229,7 +9229,13 @@ export class Brainy implements BrainyInterface { } const floorGeneration = this.generationStore.generation() - await this.storage.restoreFromDirectory(path) + // The swap runs inside the generation store's exclusive section: pending + // flush timers are disarmed and buffers discarded BEFORE any directory is + // removed, so a background flush can never write into `_system/` mid-swap + // (the ENOTEMPTY race a checkpoint stamp once hit). + await this.generationStore.runStateReplacement(() => + this.storage.restoreFromDirectory(path) + ) await this.generationStore.reopenAfterRestore(floorGeneration) // If the entity-id mapper is a NATIVE provider with a `rebuild()`, reload it diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 837ea90a..4002c1ba 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -3127,6 +3127,28 @@ export class GenerationStore { * are never reissued. * @param floorGeneration - The counter value before the restore. */ + /** + * @description Run a wholesale state replacement (the restore swap) + * EXCLUSIVELY: under the commit mutex, with the pending flush timer + * disarmed and the pending tier + fold-checkpoint accumulator discarded + * FIRST — so no background flush can write into `_system/` while the + * replacement is removing and swapping directories. Observed without this: + * a checkpoint stamp raced restore's directory removal and the swap died + * ENOTEMPTY mid-flight. The discarded in-memory state describes the store + * being replaced — `reopenAfterRestore` (which the caller runs next) + * rebuilds everything from the restored bytes. + */ + async runStateReplacement(replace: () => Promise): Promise { + return this.withMutex(async () => { + this.clearPendingFlushTimer() + this.pendingGens = [] + this.pendingBuffer.clear() + this.checkpointDirtyNouns = new Set() + this.checkpointDirtyVerbs = new Set() + await replace() + }) + } + async reopenAfterRestore(floorGeneration: number): Promise { await this.withMutex(async () => { this.deltaCache.clear() @@ -3139,6 +3161,27 @@ export class GenerationStore { this.clearPendingFlushTimer() this.pendingGens = [] this.pendingBuffer.clear() + // The fold-checkpoint accumulator described the replaced state too. + this.checkpointDirtyNouns = new Set() + this.checkpointDirtyVerbs = new Set() + this.foldCheckpointChainValid = false + this.foldCheckpoint = 0 + // A RESTORE IS AN UNCLEAN EVENT, by construction: the snapshot's files + // were just bulk-copied WITHOUT per-file fsync, so a power cut here can + // tear them — yet the snapshot may CARRY the source brain's + // clean-shutdown marker and fold checkpoint, which would together + // suppress exactly the recovery fold that cures such a tear. Delete + // both BEFORE reopening: the open below then treats the store as + // uncleanly shut, folds the restored log into canonical, barrier-syncs + // what it re-applied, and stamps a FRESH checkpoint — the restored + // state becomes durably founded at restore time instead of inheriting + // the source brain's assertions about bytes this disk never synced. + try { + await this.storage.deleteRawObject(CLEAN_SHUTDOWN_PATH) + } catch { /* absent is fine — same outcome */ } + try { + await this.storage.deleteRawObject(FOLD_CHECKPOINT_PATH) + } catch { /* absent is fine — fold from 0 */ } this.opened = false // open() re-reads counter/manifest and re-registers the bump hook. await this.open() diff --git a/tests/integration/fold-checkpoint-bound.test.ts b/tests/integration/fold-checkpoint-bound.test.ts index ce074dcf..60bcce5e 100644 --- a/tests/integration/fold-checkpoint-bound.test.ts +++ b/tests/integration/fold-checkpoint-bound.test.ts @@ -172,6 +172,41 @@ describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], nev expect(readCheckpoint(dir)).toBeNull() }, 120000) + it('restore is an UNCLEAN event: the snapshot’s stamps do not survive — the reopen fold re-founds and re-stamps the restored state', async () => { + const dir = trackDir() + const brain = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(brain) + const idA = await brain.add({ data: 'survives the restore', type: NounType.Document, metadata: { n: 1 } }) + await brain.flush() + + const snapDir = join(trackDir(), 'snap') + const db = brain.now() + await (db as unknown as { persist(p: string): Promise }).persist(snapDir) + await (db as unknown as { release(): Promise }).release() + + // Advance the live brain past the snapshot: a later write, a later flush, + // a later checkpoint stamp — none of which may survive the restore. + const idB = await brain.add({ data: 'must not survive', type: NounType.Document, metadata: { n: 2 } }) + await brain.flush() + const stampBeforeRestore = readCheckpoint(dir) + expect(stampBeforeRestore).toBe(committedOf(brain)) + + // Unflushed traffic in flight at restore time — the quiesced swap discards + // it under the mutex instead of letting its flush timer race the swap + // (the ENOTEMPTY class). + await brain.add({ data: 'in-flight at restore', type: NounType.Document, metadata: { n: 3 } }) + await brain.restore(snapDir, { confirm: true }) + + expect(await brain.get(idA), 'snapshot state restored').not.toBeNull() + expect(await brain.get(idB), 'post-snapshot state replaced').toBeNull() + // The stamp on disk is the REOPEN FOLD's fresh assertion about the + // restored (and now barrier-synced) bytes — at the restored watermark, + // strictly below the pre-restore stamp that must not survive. + const stampAfterRestore = readCheckpoint(dir) + expect(stampAfterRestore).toBe(committedOf(brain)) + expect(stampAfterRestore!).toBeLessThan(stampBeforeRestore!) + }, 120000) + it('a delete rides the barrier: the tombstoned id is in the synced set and the stamp advances past it', async () => { const dir = trackDir() const brain = await openBrain(dir, { logAuthority: 'adopt' }) From 7d3c8696d342a07ac35e9d1e055489ccc7f386b8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 13 Aug 2026 15:39:57 -0700 Subject: [PATCH 005/102] =?UTF-8?q?docs(releases):=20the=2010.1.0=20consum?= =?UTF-8?q?er=20entry=20=E2=80=94=20bounded=20recovery,=20restore=20foundi?= =?UTF-8?q?ng,=20the=20two=20write-path=20cures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index df05a81e..8116db0e 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,45 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.1.0 — 2026-08-13 (the bounded-recovery and write-path-cure release) + +The theme: **crash recovery is bounded, restores are durably founded, and two +production-reported write-path defects are cured at their roots.** Ships together +with the matching native accelerator version; adopt as a pair. + +- **Bounded crash recovery (the fold-checkpoint bound).** Recovery after an unclean + shutdown now replays only the log segment above a durably-stamped checkpoint + instead of the whole log. The checkpoint advances only after a canonical-sync + barrier makes every touched record durable (deletes included), so the bound can + lag but can never overstate durability. Existing stores converge automatically at + their first recovery — zero operator steps; recovery cost stops scaling with + store age. +- **Restores are unclean events, by construction.** `restore()` now runs its swap + fully quiesced (no background flush can race the directory replacement — a + consumer-reported `ENOTEMPTY` crash class is dead), and a snapshot's durability + stamps never survive the restore: the reopen folds the restored log, re-syncs + what it re-applied, and stamps fresh. Restored state is durably founded at + restore time instead of inheriting assertions about bytes the disk never synced. +- **Write-path cures from a production report.** (1) Log pad-frame construction is + total — a size-class boundary hole could previously kill a sync with "pad frame + not constructible". (2) The at-ack sync-failure compensation now splits by phase: + the generation counter can never re-mint a number the log may already carry, so + the non-monotonic append refusal loop reported by a downstream deployment cannot + recur. Both pinned with the reporter's exact shapes. +- **Operator-truthful sparse queries.** `where` on a field no store row has ever + carried now serves the honest answer (`eq`/`in`/range → empty; `ne`/`exists:false` + → all rows; `exists:true` → empty) with a throttled did-you-mean warning, instead + of refusing. `orderBy` on unknown fields and ambiguous spellings keep their typed + refusals. +- **Cross-package error identity.** `UnresolvableFieldError` thrown across package + boundaries is re-normalized so `instanceof` checks in consuming applications + match regardless of duplicated dependency trees. +- Release tooling: publishes now push the tag before the branch (the publish + workflow can no longer queue behind a redundant CI run) and verify registry + byte-identity with a propagation-tolerant raw-registry probe. + +--- + ## v10.0.0 — 2026-08-10 (the write-path and lifecycle release) The theme: **writes ack fast and honestly, startup adopts instead of rebuilding, and From 3915180f7b14c89b45bc5cd588a5bf307bed17c6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 13 Aug 2026 15:40:24 -0700 Subject: [PATCH 006/102] chore(release): 10.1.0 --- CHANGELOG.md | 9 +++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5482bf3f..47a767ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13) + +- docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) +- fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) +- feat(recovery): the fold-checkpoint bound — crash folds (checkpoint, head], never the whole log twice (ff43de1a) +- fix(log): pad-frame construction is total; the at-ack sync-failure compensation splits by phase — a production adoption's two write-path defects, cured at their roots (cbe34d11) +- feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) + + ### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12) - fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) diff --git a/package-lock.json b/package-lock.json index 6193a630..8219ed2c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.0.0", + "version": "10.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.0.0", + "version": "10.1.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 7b93cdd7..6dc73761 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.0.0", + "version": "10.1.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From a5a1883819f1d1661dadf369c15c045d3df7e7b9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 17 Aug 2026 12:53:04 -0700 Subject: [PATCH 007/102] =?UTF-8?q?fix(adoption):=20the=20baseline=20backf?= =?UTF-8?q?ill=20runs=20to=20completion=20=E2=80=94=20one=20call=20adopts?= =?UTF-8?q?=20a=20pre-log=20baseline=20of=20any=20size?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production brain with a 12.7k-row pre-log baseline advanced exactly 800 rows per adoptLogAuthority() call (a five-pass ceiling × the oracle's 200-row listing cap), refused the flip, and sat tree-authoritative for hours across restarts. The bound was sized for drift, never for a baseline. Now: the adoption path runs the oracle uncapped so ONE scan yields the ENTIRE curable set, every pass cures all of it, and the loop runs to completion with the no-progress guard as its only stop. Pace rides the write path (one full-brain scan amortizes over thousands of cures, not two hundred): 1,000 drifted rows adopt green in one call in ~10s. Progress is narrated for a live operator. The wire report keeps its 200-row cap. Pinned: a baseline above the old ceiling adopts green in a single call. --- src/brainy.ts | 51 ++++++++-- src/db/logAuthority.ts | 11 ++- .../integration/adopt-large-baseline.test.ts | 92 +++++++++++++++++++ 3 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 tests/integration/adopt-large-baseline.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index d7313855..addb8bc2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8208,10 +8208,20 @@ export class Brainy implements BrainyInterface { * (digests, never bodies). */ async verifyLogAuthority(): Promise { + return this.runOracle() + } + + /** + * The oracle run behind {@link Brainy.verifyLogAuthority}; the adoption + * backfill calls it with `listAll` so one scan yields the ENTIRE curable + * mismatch set instead of the wire-capped first 200. + */ + private async runOracle(options?: { listAll?: boolean }): Promise { await this.ensureInitialized() return runLogCompletenessOracle({ storage: this.storage as unknown as LogAuthorityStorage, scanFacts: () => this.scanFacts(), + ...(options?.listAll ? { mismatchListCap: Number.POSITIVE_INFINITY } : {}), // Both sides normalize to ENTITY TRUTH before digesting: canonical // wrappers denormalize HNSW residue (connections/level) the log never // carries — digesting it would fake state-differs on any nonzero-level @@ -8256,7 +8266,7 @@ export class Brainy implements BrainyInterface { /** The adoption body — see {@link Brainy.adoptLogAuthority} (which owns the * fold-checkpoint bootstrap arm/disarm around it). */ private async adoptLogAuthorityInner(): Promise { - let report = await this.verifyLogAuthority() + let report = await this.runOracle({ listAll: true }) // BASELINE BACKFILL: curable divergences are rows whose CANONICAL truth // simply never reached the log — pre-log records (e.g. the generation-0 @@ -8268,8 +8278,18 @@ export class Brainy implements BrainyInterface { // Log-AHEAD divergences (log-live-canonical-absent / // log-tombstone-canonical-present) are NOT curable by backfill — the // log claims things the witness denies — and refuse loudly below. + // + // RUNS TO COMPLETION. Each pass sees the ENTIRE curable set (the oracle + // is run uncapped here) and cures all of it, so a pre-log baseline of + // any size adopts in ONE call — the only stop is the no-progress guard. + // A production brain with a 12.7k-row baseline once advanced exactly + // 800 rows per call (a five-pass ceiling × the 200-row wire cap) and sat + // tree-authoritative for hours; the bound was sized for drift, never + // for a baseline. Pace rides the write path now: one full-brain scan + // per pass amortizes over thousands of cures, not two hundred. let passes = 0 - while (report.verdict === 'red' && passes < 5) { + for (;;) { + if (report.verdict !== 'red') break passes++ const curable = report.mismatches.filter( (m) => m.reason === 'pre-log-record' || m.reason === 'state-differs' @@ -8290,9 +8310,19 @@ export class Brainy implements BrainyInterface { `[Brainy] adoptLogAuthority: baseline backfill pass ${passes} — re-committing ` + `${curable.length} row(s) whose canonical truth never reached the log` ) + // Progress narration for a live operator: a large baseline is minutes + // of visible motion, never a silent wait. + const narrateEvery = curable.length >= 2000 ? 1000 : curable.length >= 400 ? 200 : 0 + let cured = 0 for (const m of curable) { const raw = await this.storage.readNounRaw(m.id) if (raw.metadata === null && raw.vector === null) continue // vanished since the scan + cured++ + if (narrateEvery > 0 && cured % narrateEvery === 0) { + prodLog.info( + `[Brainy] adoptLogAuthority: backfill pass ${passes} — ${cured}/${curable.length} rows re-committed` + ) + } // LAW-SHAPE RE-COMMIT: rewrite canonical as EXACTLY the wrapper the // log's reconstruction produces (the hydration law: denormalized // enumeration fields derived from the metadata leg + the embedding @@ -8330,12 +8360,11 @@ export class Brainy implements BrainyInterface { }) }) } - const next = await this.verifyLogAuthority() - if ( - next.verdict === 'red' && - next.mismatches.length >= report.mismatches.length && - !report.mismatchListTruncated - ) { + const next = await this.runOracle({ listAll: true }) + // THE ONLY STOP: no progress. With uncapped listings both counts are + // exact, so "not fewer mismatches than before" means the cure could + // not express this divergence — refuse to spin, name it. + if (next.verdict === 'red' && next.mismatches.length >= report.mismatches.length) { throw new Error( `adoptLogAuthority(): baseline backfill made no progress ` + `(${report.mismatches.length} → ${next.mismatches.length} mismatches; first: ` + @@ -8345,6 +8374,12 @@ export class Brainy implements BrainyInterface { } report = next } + if (passes > 0) { + prodLog.info( + `[Brainy] adoptLogAuthority: baseline backfill complete in ${passes} pass(es) — ` + + `oracle ${report.verdict}, ${report.nounsChecked} noun(s) checked` + ) + } this._logAuthority = await flipToLogAuthority( this.storage as unknown as LogAuthorityStorage, diff --git a/src/db/logAuthority.ts b/src/db/logAuthority.ts index 0703d11f..b63ae715 100644 --- a/src/db/logAuthority.ts +++ b/src/db/logAuthority.ts @@ -165,7 +165,16 @@ export async function runLogCompletenessOracle(args: { getVerbs?: (opts: { pagination: { limit: number; offset?: number; cursor?: string } }) => Promise<{ items: unknown[]; hasMore?: boolean; nextCursor?: string }> + /** + * Cap on the LISTED mismatches (counts are always complete). Defaults to + * the wire-friendly {@link MISMATCH_LIST_CAP}; the adoption backfill passes + * `Infinity` so ONE scan yields the ENTIRE curable set — a production + * brain with a 12.7k-row pre-log baseline once advanced only 800 rows per + * adoption call because each pass could see (and cure) at most 200. + */ + mismatchListCap?: number }): Promise { + const listCap = args.mismatchListCap ?? MISMATCH_LIST_CAP const report: OracleReport = { verdict: 'red', generationsScanned: 0, @@ -176,7 +185,7 @@ export async function runLogCompletenessOracle(args: { mismatchListTruncated: false } const addMismatch = (m: OracleMismatch): void => { - if (report.mismatches.length < MISMATCH_LIST_CAP) report.mismatches.push(m) + if (report.mismatches.length < listCap) report.mismatches.push(m) else report.mismatchListTruncated = true } diff --git a/tests/integration/adopt-large-baseline.test.ts b/tests/integration/adopt-large-baseline.test.ts new file mode 100644 index 00000000..11a3c803 --- /dev/null +++ b/tests/integration/adopt-large-baseline.test.ts @@ -0,0 +1,92 @@ +/** + * @module tests/integration/adopt-large-baseline + * @description Adoption runs the baseline backfill TO COMPLETION in one call. + * A production brain with a 12.7k-row pre-log baseline once advanced exactly + * 800 rows per `adoptLogAuthority()` call (a five-pass ceiling × the oracle's + * 200-row listing cap), refused the flip, and sat tree-authoritative for + * hours across restarts. The pin: a baseline larger than that old ceiling + * — every row oracle-visible as `state-differs` drift — adopts GREEN in a + * SINGLE call, and the row count proves the whole set was cured, not a page. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +type RawBox = { + storage: { + readNounRaw(id: string): Promise<{ metadata: unknown; vector: unknown }> + writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise + } +} + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +describe('adoption backfill runs to completion', () => { + it('a pre-log baseline larger than the old 800-row ceiling adopts GREEN in ONE call', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-large-baseline-')) + dirs.push(dir) + const brain = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false, + logAuthority: 'defer' + }) + await brain.init() + brains.push(brain) + + // Above the old ceiling (5 passes × 200 = 800): every row must be cured + // in the one call for the flip to be legal. + const ROWS = 1000 + const ids: string[] = [] + for (let i = 0; i < ROWS; i++) { + ids.push( + await brain.add({ + data: `baseline row ${i}`, + type: NounType.Document, + metadata: { i }, + vector: Array.from({ length: 384 }, (_, k) => ((i + k) % 7) / 7) + }) + ) + } + await brain.flush() + + // Manufacture the production shape on EVERY row: pre-hydration-law drift + // (a stored wrapper whose denormalized fields disagree with its own + // metadata leg) — each is a curable `state-differs` mismatch, so the + // oracle's full curable set is ROWS, well past any per-pass page. + const storage = (brain as unknown as RawBox).storage + for (const id of ids) { + const raw = await storage.readNounRaw(id) + const wrapper = raw.vector as Record + await storage.writeNounRaw(id, { + metadata: raw.metadata, + vector: { ...wrapper, noun: 'thing', legacyField: 'pre-law residue' } + }) + } + const before = await brain.verifyLogAuthority() + expect(before.verdict, 'the whole baseline is oracle-red').toBe('red') + // The wire report is capped at 200 — the truncation flag is what the old + // loop bounded itself on; the cure path no longer reads through it. + expect(before.mismatchListTruncated).toBe(true) + + // THE PIN: one call, green, log-authoritative — no restarts, no loop. + const report = await brain.adoptLogAuthority() + expect(report.verdict).toBe('green') + expect(brain.logAuthority().authority).toBe('log') + expect(report.nounsChecked).toBeGreaterThanOrEqual(ROWS) + + // Nothing degraded: a sample of rows still serves with intact metadata. + for (const id of [ids[0], ids[499], ids[ROWS - 1]]) { + const row = await brain.get(id) + expect(row).not.toBeNull() + expect(typeof (row!.metadata as { i: number }).i).toBe('number') + } + }, 600000) +}) From b17fdc8e36b6bd53a34a6f475cbb567720a8ae71 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 17 Aug 2026 13:48:07 -0700 Subject: [PATCH 008/102] =?UTF-8?q?ci:=20the=20correctness=20plant=20runs?= =?UTF-8?q?=20integration=20+=20conformance=20on=20every=20push=20?= =?UTF-8?q?=E2=80=94=20a=20release=20never=20waits=20on=20a=20second=20mac?= =?UTF-8?q?hine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .forgejo/workflows/ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index fec679a8..5e93cd96 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -27,6 +27,22 @@ jobs: - run: npm ci - run: npm run test:unit + # The correctness plant's full gate: integration + conformance run here on + # dedicated iron, on every push, so a release never depends on any other + # machine being up. Verdicts live in this run's log (never inferred). + integration: + name: Integration + conformance (Node 22) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - run: npm ci + - run: npm run test:ci-integration + - run: npx vitest run tests/conformance + bun: name: Bun (latest) runs-on: ubuntu-latest From 97538e1f0796b82b05cc69275e1df4610bdb5734 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 17 Aug 2026 14:44:59 -0700 Subject: [PATCH 009/102] =?UTF-8?q?docs(releases):=20the=2010.2.0=20consum?= =?UTF-8?q?er=20entry=20=E2=80=94=20adoption=20completes=20in=20one=20call?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 8116db0e..602b84e4 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,30 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.2.0 — 2026-08-17 (adoption completes in one call) + +One fix, headline-sized for large stores. Pairs with the same native accelerator +version as 10.1.0 — no accelerator bump needed. + +- **The adoption backfill runs to completion.** Adopting the crash-safe storage + authority first re-commits every row the log never saw (a one-time baseline + backfill). That backfill had a fixed ceiling of 800 rows per + `adoptLogAuthority()` call — sized for small drift, not for a large pre-existing + store — so a store with a 12,700-row baseline advanced 800 rows per call and + stayed on the prior authority across restarts (a production deployment's + report). Now one call adopts a baseline of any size: the backfill sees the + entire curable set at once, cures all of it, and loops only until green — the + no-progress guard is the sole stop. Pace rides the write path (~100 rows/s + measured end to end, versus ~1.7 rows/s under the old page-per-scan shape), + and progress is narrated so an operator watching a live service sees motion. + Stores that already adopted are unaffected; stores still on the prior authority + flip in a single call on their next open or on an explicit + `adoptLogAuthority()`. +- Verification report unchanged on the wire (still lists at most 200 mismatches; + counts remain complete) — only the adoption path reads the full set. + +--- + ## v10.1.0 — 2026-08-13 (the bounded-recovery and write-path-cure release) The theme: **crash recovery is bounded, restores are durably founded, and two From f4653e47c906ecc33314afb7e6a77e0449dbf5b6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 17 Aug 2026 14:45:20 -0700 Subject: [PATCH 010/102] chore(release): 10.2.0 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47a767ec..8e91a6ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17) + +- docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) +- ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) +- fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) + + ### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13) - docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) diff --git a/package-lock.json b/package-lock.json index 8219ed2c..e8c238f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.1.0", + "version": "10.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.1.0", + "version": "10.2.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 6dc73761..a366f42f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.1.0", + "version": "10.2.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 9ac9e70686eacdbf70470bf05ebf728faf034bc4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 17 Aug 2026 16:21:25 -0700 Subject: [PATCH 011/102] feat(log): system commits carry their origin; the attested per-id reconcile door MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two consumer-driven cures sharing one stamp. (1) TX-LOG ORIGIN: engine- originated commits stamp an optional origin on their tx-log entry AND the commit fact's meta — 'system:embed-landing' (the deferred vector landing), 'system:adoption-backfill' (baseline re-commits), 'system:reconcile'. A downstream activity feed showed a double tick because the landing commit was indistinguishable from a user save, and the consumer rightly refused a time-window collapse as a quiet loss; feeds now filter on fact. User writes stay unstamped — absent origin is the user shape, every existing consumer unchanged. (2) reconcileLogDivergence(id, {attest}): the human's door for log-live-canonical-absent, the one class adoption refuses by design because a lost-tombstone deletion is indistinguishable from canonical loss. 'deleted' mints the missing tombstone (history keeps the earlier live record); 'restore' folds the log's only copy back into canonical; wrong- class calls refuse typed with nothing written. Loud, narrated, single-row, origin-stamped. From a production adoption's one surviving divergence. --- src/brainy.ts | 140 ++++++++++++++++- src/db/generationStore.ts | 22 ++- src/db/types.ts | 11 ++ .../txlog-origin-and-reconcile.test.ts | 142 ++++++++++++++++++ 4 files changed, 308 insertions(+), 7 deletions(-) create mode 100644 tests/integration/txlog-origin-and-reconcile.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index addb8bc2..d1ec144b 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -2245,7 +2245,8 @@ export class Brainy implements BrainyInterface { }, undefined, undefined, - [{ type: 'embed.landed', id, vector: newVector }] + [{ type: 'embed.landed', id, vector: newVector }], + 'system:embed-landing' ) this.clearPendingEmbed(id) } catch (err) { @@ -2479,7 +2480,8 @@ export class Brainy implements BrainyInterface { run: TransactionFunction, precommit?: (before: CommitBeforeImages) => void, pendingEvents?: PendingChangeEvent[], - records?: FactMarkerRecord[] + records?: FactMarkerRecord[], + origin?: string ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> { // Change-feed capture: when this write will emit, hold a reference to the // commit's before-images so `remove` events can carry the record's last @@ -2542,6 +2544,7 @@ export class Brainy implements BrainyInterface { touched, precommit: captureAndCheck, ...(records && records.length > 0 ? { records } : {}), + ...(origin ? { origin } : {}), execute: () => this.transactionManager.executeTransaction(run, { timeout: transactTimeoutBudget( @@ -8358,7 +8361,7 @@ export class Brainy implements BrainyInterface { } } }) - }) + }, undefined, undefined, undefined, 'system:adoption-backfill') } const next = await this.runOracle({ listAll: true }) // THE ONLY STOP: no progress. With uncapped listings both counts are @@ -8392,6 +8395,137 @@ export class Brainy implements BrainyInterface { return report } + /** + * @description THE ATTESTED PER-ID RECONCILE DOOR for the one divergence + * class the adoption backfill refuses BY DESIGN: `log-live-canonical-absent` + * — the log holds a live record for a row the canonical tree says does not + * exist. The engine cannot tell a legitimate pre-log deletion (the log + * missed the tombstone — the deferred-durability-era ack-window class) from + * canonical LOSS (the log holds the only surviving copy); auto-curing would + * silently destroy data in one of the two readings. A HUMAN attests which: + * + * - `attest: 'deleted'` — the row was legitimately deleted; mint the + * tombstone fact the log always lacked (canonical stays absent). The + * log's history keeps the old live record — as-of reads before the + * tombstone still see it. + * - `attest: 'restore'` — canonical lost the row; fold the log's latest + * after-image back into canonical (both sides now agree it lives). + * + * Loud, narrated, single-row, and stamped `origin: 'system:reconcile'` on + * both the tx-log entry and the commit fact. Refuses (typed) when the id's + * log and canonical already agree, when `restore` is attested but the log + * holds no record, and when canonical is PRESENT-but-different (that is + * `state-differs` — `adoptLogAuthority()`'s backfill owns it). + * + * @param id - The single entity id to reconcile. + * @param options.attest - The human's word on which reading is true. + * @returns What was done and the generation that recorded it. + * @throws When the divergence is not the attested class (nothing is written). + */ + async reconcileLogDivergence( + id: string, + options: { attest: 'deleted' | 'restore' } + ): Promise<{ reconciled: 'tombstoned' | 'restored'; id: string; generation: number }> { + await this.ensureInitialized() + this.assertWritable('reconcileLogDivergence') + + // Fold the log for THIS id (one scan; a rare operator door). + const scan = this.scanFacts() + if (!scan) { + throw new Error('reconcileLogDivergence: this store has no fact log — nothing to reconcile against') + } + let logLatest: { tombstoned: boolean; record: { metadata: unknown; vector: unknown } | null } | null = null + for await (const batch of scan.batches()) { + for (const fact of batch.facts) { + for (const op of fact.ops) { + if (op.kind === 'noun' && op.id === id) { + logLatest = + op.record === null + ? { tombstoned: true, record: null } + : { tombstoned: false, record: { metadata: op.record.metadata, vector: op.record.vector } } + } + } + } + } + const canonical = await this.storage.readNounRaw(id) + const canonicalAbsent = canonical.metadata === null && canonical.vector === null + + // Only the log-live + canonical-absent shape passes; everything else + // names its actual state and the door that owns it. + if (!logLatest || logLatest.tombstoned) { + throw new Error( + `reconcileLogDivergence(${id}): the log's latest state is ` + + `${logLatest ? 'a tombstone' : 'no record at all'} — there is no ` + + `log-live-canonical-absent divergence here. If the oracle reports this id, ` + + `re-run verifyLogAuthority() for the current class.` + ) + } + if (!canonicalAbsent) { + throw new Error( + `reconcileLogDivergence(${id}): canonical is PRESENT — this is not the ` + + `log-live-canonical-absent class. If canonical differs from the log ` + + `(state-differs), adoptLogAuthority()'s backfill cures it; nothing was written.` + ) + } + + if (options.attest === 'deleted') { + // Mint the tombstone fact the log always lacked. writeNounRaw with null + // parts is an idempotent delete; the commit fact reads canonical back + // after execute (absent) and records the tombstone. + const receipt = await this.persistSingleOp( + { nouns: [id] }, + async (tx) => { + tx.addOperation({ + name: 'ReconcileTombstone', + execute: async () => { + await this.storage.writeNounRaw(id, { metadata: null, vector: null }) + return async () => { + // Undo of an idempotent delete of an absent row: nothing. + } + } + }) + }, + undefined, + undefined, + undefined, + 'system:reconcile' + ) + prodLog.warn( + `[Brainy] reconcileLogDivergence: ${id} attested DELETED — tombstone fact minted ` + + `at generation ${receipt.generation}; the log now agrees the row is gone ` + + `(its history keeps the earlier live record).` + ) + return { reconciled: 'tombstoned', id, generation: receipt.generation! } + } + + // attest: 'restore' — the log's copy is the survivor; fold it back. + const record = logLatest.record! + const receipt = await this.persistSingleOp( + { nouns: [id] }, + async (tx) => { + tx.addOperation({ + name: 'ReconcileRestore', + execute: async () => { + await this.storage.writeNounRaw(id, record) + return async () => { + await this.storage.writeNounRaw(id, { metadata: null, vector: null }) + } + } + }) + }, + undefined, + undefined, + undefined, + 'system:reconcile' + ) + prodLog.warn( + `[Brainy] reconcileLogDivergence: ${id} attested RESTORE — the log's latest ` + + `after-image was folded back into canonical at generation ${receipt.generation}. ` + + `Derived indexes reconcile at next open/repairIndex; the row serves from canonical now.` + ) + return { reconciled: 'restored', id, generation: receipt.generation! } + } + /** * @description Read the reified transaction log — one entry per committed * generation, carrying the committed generation, the commit timestamp, and diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 4002c1ba..7439a025 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -428,7 +428,13 @@ export class GenerationStore { private pendingGens: number[] = [] private readonly pendingBuffer = new Map< number, - { nouns: Map; verbs: Map; timestamp: number } + { + nouns: Map + verbs: Map + timestamp: number + /** Engine-origin stamp for the tx-log entry (absent = user write). */ + origin?: string + } >() /** Pending timer-coalesced flush handle (cleared on flush/close). */ private pendingFlushTimer: ReturnType | null = null @@ -1642,6 +1648,12 @@ export class GenerationStore { * surfacing that honestly. */ records?: FactMarkerRecord[] + /** + * Engine-origin stamp (`'system:embed-landing'`, `'system:adoption-backfill'`, + * `'system:reconcile'`). Rides the tx-log entry AND the commit fact's meta, + * so both records agree about WHO committed. Absent = user write. + */ + origin?: string }): Promise<{ generation: number; timestamp: number; degraded?: string[] }> { return this.withMutex(async () => { // Refuse to accept a write whose history we cannot make durable: if the @@ -1710,7 +1722,7 @@ export class GenerationStore { // incomplete for these ids until the next rebuild/repairIndex (the // egress guard prevents wrong results meanwhile). Loud, honest, // no double-write. - this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) + this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp, ...(args.origin ? { origin: args.origin } : {}) }) this.pendingGens.push(gen) this.extendChains(gen, nouns, verbs) // The adopted generation is committed — it gets its fact like any @@ -1723,6 +1735,7 @@ export class GenerationStore { timestamp, nouns, verbs, + ...(args.origin ? { meta: { origin: args.origin } } : {}), ...(args.records && args.records.length > 0 ? { records: args.records } : {}) }) ) @@ -1763,7 +1776,7 @@ export class GenerationStore { if (this.commitFaultInjector) this.commitFaultInjector('singleop-after-execute') // Buffer the pending generation + make it instantly visible to reads. - this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp }) + this.pendingBuffer.set(gen, { nouns: nounBefore, verbs: verbBefore, timestamp, ...(args.origin ? { origin: args.origin } : {}) }) this.pendingGens.push(gen) this.extendChains(gen, nouns, verbs) // Fact log (dual-write): the acked write's AFTER-IMAGE fact, appended @@ -1802,6 +1815,7 @@ export class GenerationStore { timestamp, nouns, verbs, + ...(args.origin ? { meta: { origin: args.origin } } : {}), ...(args.records && args.records.length > 0 ? { records: args.records } : {}) }) ) @@ -1958,7 +1972,7 @@ export class GenerationStore { const deltaPath = `${dir}/tx.json` await this.storage.writeRawObject(deltaPath, delta) stagedPaths.push(deltaPath) - logEntries.push({ generation: gen, timestamp: buf.timestamp }) + logEntries.push({ generation: gen, timestamp: buf.timestamp, ...(buf.origin ? { origin: buf.origin } : {}) }) } // Test-only crash simulation. A crash here must cost only the window's diff --git a/src/db/types.ts b/src/db/types.ts index 2c7eab8f..866ca47f 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -412,6 +412,17 @@ export interface TxLogEntry { timestamp: number /** Transaction metadata, when supplied to `transact()`. */ meta?: Record + /** + * WHO committed. Absent = a user write (every pre-existing consumer's + * reading stays exact). Engine-originated commits stamp themselves — + * `'system:embed-landing'` (the deferred vector landing), + * `'system:adoption-backfill'` (baseline re-commits), `'system:reconcile'` + * (the attested per-id divergence door) — so activity feeds can filter on + * fact instead of collapsing near-in-time entries (a consumer refused that + * heuristic as a quiet loss, correctly; this field is the honest cure). + * The same stamp rides the commit fact's meta, so log and tx-log agree. + */ + origin?: string } // ============================================================================ diff --git a/tests/integration/txlog-origin-and-reconcile.test.ts b/tests/integration/txlog-origin-and-reconcile.test.ts new file mode 100644 index 00000000..fbe05fcf --- /dev/null +++ b/tests/integration/txlog-origin-and-reconcile.test.ts @@ -0,0 +1,142 @@ +/** + * @module tests/integration/txlog-origin-and-reconcile + * @description Two consumer-driven cures, pinned together because they share + * the origin stamp: + * + * 1. TX-LOG ORIGIN — engine-originated commits stamp `origin` on their + * tx-log entry (and the commit fact's meta) so activity feeds filter on + * fact: a downstream feed showed a "double tick" because the deferred + * vector-landing commit was indistinguishable from a user save, and the + * consumer rightly refused a time-window collapse as a quiet loss. User + * writes stay UNSTAMPED (absent origin) — the pre-existing reading of + * every consumer is exact. + * + * 2. THE RECONCILE DOOR — `log-live-canonical-absent` refuses auto-cure by + * design (a legitimate lost-tombstone deletion is indistinguishable from + * canonical loss); `reconcileLogDivergence(id, {attest})` is the human's + * door: 'deleted' mints the missing tombstone, 'restore' folds the log's + * copy back, wrong-class calls refuse typed with nothing written. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +type RawBox = { + storage: { + readNounRaw(id: string): Promise<{ metadata: unknown; vector: unknown }> + writeNounRaw(id: string, r: { metadata: unknown; vector: unknown }): Promise + } +} + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +async function fsBrain(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-origin-reconcile-')) + dirs.push(dir) + const brain = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false + }) + await brain.init() + brains.push(brain) + return brain +} + +describe('tx-log origin stamp', () => { + it('the deferred-embed landing commit is stamped system:embed-landing; the user write is not', async () => { + const brain = await fsBrain() + await brain.add({ + data: 'a row whose vector lands later', + type: NounType.Document, + metadata: { k: 1 }, + deferEmbedding: true + }) + await brain.awaitPendingEmbeds() + await brain.flush() + + const entries = await brain.transactionLog() + const system = entries.filter((e) => (e as { origin?: string }).origin === 'system:embed-landing') + const user = entries.filter((e) => !(e as { origin?: string }).origin) + expect(system.length, 'the landing commit is stamped').toBeGreaterThanOrEqual(1) + expect(user.length, 'the user add stays unstamped').toBeGreaterThanOrEqual(1) + // The feed cure in one line: filtering !origin removes the double tick. + expect(user.length).toBeLessThan(entries.length) + }, 120000) +}) + +describe('reconcileLogDivergence — the attested door', () => { + /** Manufacture the class: a live log record whose canonical row is gone. */ + async function manufactureDivergence(brain: Brainy): Promise { + const id = await brain.add({ + data: 'pre-era row whose deletion the log never saw', + type: NounType.Document, + metadata: { era: 'pre-spine' } + }) + await brain.flush() + // Delete canonical BEHIND the log's back (raw write, no generation) — + // exactly the shape a deferred-durability-era crash left behind. + const storage = (brain as unknown as RawBox).storage + await storage.writeNounRaw(id, { metadata: null, vector: null }) + return id + } + + it("attest:'deleted' mints the missing tombstone — the oracle goes green and the commit is stamped system:reconcile", async () => { + const brain = await fsBrain() + const id = await manufactureDivergence(brain) + const before = await brain.verifyLogAuthority() + expect( + before.mismatches.some((m) => m.id === id && m.reason === 'log-live-canonical-absent'), + 'the manufactured divergence is oracle-visible as the refused class' + ).toBe(true) + + const result = await brain.reconcileLogDivergence(id, { attest: 'deleted' }) + expect(result.reconciled).toBe('tombstoned') + + const after = await brain.verifyLogAuthority() + expect(after.mismatches.some((m) => m.id === id), 'the id no longer diverges').toBe(false) + expect(await brain.get(id), 'canonical stays absent').toBeNull() + + await brain.flush() + const entries = await brain.transactionLog() + expect( + entries.some((e) => (e as { origin?: string }).origin === 'system:reconcile'), + 'the reconcile commit is origin-stamped' + ).toBe(true) + }, 120000) + + it("attest:'restore' folds the log's copy back into canonical", async () => { + const brain = await fsBrain() + const id = await manufactureDivergence(brain) + + const result = await brain.reconcileLogDivergence(id, { attest: 'restore' }) + expect(result.reconciled).toBe('restored') + + const row = await brain.get(id) + expect(row, 'the log’s only copy lives again').not.toBeNull() + expect((row!.metadata as { era: string }).era).toBe('pre-spine') + expect((await brain.verifyLogAuthority()).mismatches.some((m) => m.id === id)).toBe(false) + }, 120000) + + it('wrong-class calls refuse typed with nothing written', async () => { + const brain = await fsBrain() + const id = await brain.add({ data: 'healthy row', type: NounType.Document, metadata: { n: 1 } }) + await brain.flush() + // Canonical present + log agrees: not the class — refuse, name the state. + await expect(brain.reconcileLogDivergence(id, { attest: 'deleted' })).rejects.toThrow( + /canonical is PRESENT/ + ) + expect(await brain.get(id), 'nothing was written').not.toBeNull() + // Unknown id: no log record at all — refuse, name it. + await expect( + brain.reconcileLogDivergence('00000000-0000-7000-8000-00000000dead', { attest: 'restore' }) + ).rejects.toThrow(/no record at all/) + }, 120000) +}) From 292e7c0406e49fc1663cbcdc8d6f75996bf8e102 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 17 Aug 2026 16:26:41 -0700 Subject: [PATCH 012/102] fix(locks): live writers are never auto-evicted; evicted writers are fenced at every commit barrier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The production dev-store split-brain (two live writers alternating a store's id-mapper between two internally-consistent truths), cured at all three of its roots. (1) STALENESS REQUIRES PID-DEATH: the old rule evicted on heartbeat age alone, so a >60s event-loop stall (debugger pause, GC, heavy sync work) handed the lock to a second opener while the first kept writing; a live process is now never auto-evicted — a wedged-but-alive holder is the operator's call via {force:true}, and the heartbeat stays for observability. (2) THE CLAIM IS ATOMIC: writeFile(wx)'s open→write→close left an empty-file window a concurrent opener could read as torn, unlink a LIVE claim, and take the lock; the claim is now tmp-write + hard-link — the lock appears with its full contents in one step. (3) THE FENCE: every flush commit and transact barrier verifies lock ownership first (one small read per window) — a forced-out or lock-deleted writer fails typed (BRAINY_WRITER_FENCED) before a single staged byte or manifest advance, instead of writing on unaware. Pinned: live-with-ancient-heartbeat refuses typed; dead-PID self-clears narrated; a forced-out writer's flush and transact both fence, advancing nothing. Requested by a downstream team as single-writer guard or loud lockout — this is both. --- src/db/generationStore.ts | 9 ++ src/db/types.ts | 10 ++ src/storage/adapters/fileSystemStorage.ts | 71 ++++++++-- src/storage/baseStorage.ts | 12 ++ tests/integration/writer-lock-fencing.test.ts | 131 ++++++++++++++++++ 5 files changed, 225 insertions(+), 8 deletions(-) create mode 100644 tests/integration/writer-lock-fencing.test.ts diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 7439a025..065b3659 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -1394,6 +1394,10 @@ export class GenerationStore { // The transaction's entire canonical footprint is now durable, so the // counter/manifest advance below can never outrun the entity bytes. await this.storage.flushWriteBarrier?.() + // THE FENCE (transact leg): verify lock ownership before the commit + // point — an aborted-by-fence transact rolls back cleanly through the + // catch below; a fenced writer must never advance counter or manifest. + await this.storage.assertWriterFenceHeld?.() faultPoint('after-execute') // Fact log (dual-write): append + fsync this generation's AFTER-IMAGE @@ -1915,6 +1919,11 @@ export class GenerationStore { private async flushPendingSingleOpsUnlocked(): Promise { return this.withMutex(async () => { if (this.pendingGens.length === 0) return + // THE FENCE: an evicted writer (force-takeover, removed lock) must fail + // HERE, before a single staged byte or manifest advance — writing on + // after eviction is how split-brain stores are made. One small read + // per flush window. + await this.storage.assertWriterFenceHeld?.() this.clearPendingFlushTimer() const gens = [...this.pendingGens].sort((a, b) => a - b) diff --git a/src/db/types.ts b/src/db/types.ts index 866ca47f..363de086 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -485,6 +485,16 @@ export interface GenerationStorage { */ syncEntityCanonical?(nouns: string[], verbs: string[]): Promise + /** + * OPTIONAL writer fence: throw `BRAINY_WRITER_FENCED` when this instance + * no longer owns the store's writer lock (an operator force-takeover or a + * removed lock file). Called at every flush commit and transact barrier — + * one small read per commit window — so an evicted writer fails loudly on + * its next commit instead of split-braining the store. Adapters without a + * cross-process lock model omit it. + */ + assertWriterFenceHeld?(): Promise + /** Read an entity's raw stored metadata+vector objects. */ readNounRaw(id: string): Promise<{ metadata: any | null; vector: any | null }> /** Restore an entity's raw stored objects (`null` part ⇒ delete that file). */ diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 81c6e545..8fb2d2f1 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -1920,14 +1920,24 @@ export class FileSystemStorage extends BaseStorage { rootDir: this.rootDir } - // The atomic claim: create-exclusive, so exactly ONE racer wins. + // The atomic claim: write the FULL contents to a temp file, then + // hard-link it into place — link(2) fails EEXIST if the target exists, + // and the lock file appears with its complete JSON in one atomic step. + // (The previous claim was writeFile with O_EXCL, whose open→write→close + // is NOT atomic: a concurrent opener could read the file in its empty + // window, judge it torn, unlink a LIVE claim, and take the lock — two + // live writers. The link claim leaves no empty window to misread.) + const claimTmp = `${lockFile}.claim-${myPid}-${Date.now()}` try { - await fs.promises.writeFile(lockFile, JSON.stringify(info, null, 2), { flag: 'wx' }) + await fs.promises.writeFile(claimTmp, JSON.stringify(info, null, 2)) + await fs.promises.link(claimTmp, lockFile) } catch (err: any) { if (err.code === 'EEXIST') { continue // someone else claimed between our read and create — re-evaluate } throw err + } finally { + await fs.promises.unlink(claimTmp).catch(() => {}) } this.installWriterLock(info) @@ -1972,6 +1982,44 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * THE FENCE: verify this instance still owns the writer lock before a + * commit barrier proceeds. An evicted writer (an operator's + * `{ force: true }` takeover, or an operator deleting the lock file) must + * fail LOUDLY on its next flush instead of writing on unaware — the + * unfenced evicted writer was half of a production split-brain (each + * writer flushing its own internally-consistent id-mapper snapshot, + * alternating the store between two truths). One small file read per + * flush window, never per record. No-op when this instance holds no + * writer lock (read-only opens, in-memory stores). + * + * @throws `BRAINY_WRITER_FENCED` when the lock is gone or held by another. + */ + public override async assertWriterFenceHeld(): Promise { + if (!this.writerLockInfo) return + const current = await this.readWriterLock() + if ( + current && + current.pid === this.writerLockInfo.pid && + current.hostname === this.writerLockInfo.hostname && + current.startedAt === this.writerLockInfo.startedAt + ) { + return + } + const err = new Error( + `Writer fence lost for ${this.rootDir}: this process (PID ${this.writerLockInfo.pid}) ` + + `no longer holds the writer lock — ` + + (current + ? `it is now held by PID ${current.pid} on ${current.hostname} (since ${current.startedAt}).` + : `the lock file is gone (released or removed by an operator).`) + + `\nThis instance refuses to commit further writes: a fenced-out writer continuing to ` + + `flush is how split-brain stores are made. Close this instance; if the takeover was a ` + + `mistake, close the successor and re-open.` + ) as Error & { code: string } + err.code = 'BRAINY_WRITER_FENCED' + throw err + } + /** The consumer-facing BRAINY_WRITER_LOCKED error, holder details attached. */ private writerLockedError(existing: WriterLockInfo): Error { const err = new Error( @@ -2060,18 +2108,25 @@ export class FileSystemStorage extends BaseStorage { /** * Determine whether an existing writer lock is stale (safe to overwrite). - * Same hostname and (dead PID OR heartbeat older than threshold) → stale. - * Different hostname → cannot prove stale, treat as live. + * Same hostname and DEAD PID → stale. That is the whole rule: a LIVE + * process is never auto-evicted, however old its heartbeat — a >60s + * event-loop stall (debugger pause, GC, heavy sync work) is a slow writer, + * not a dead one, and heartbeat-age eviction of live writers was the + * dominant mechanism behind a production split-brain (two live unaware + * writers alternating a store's id-mapper between two truths). A holder + * that LOOKS alive but is truly wedged is the operator's call via + * `{ force: true }` — and the fence check on every flush + * ({@link assertWriterFenceHeld}) guarantees a forced-out holder fails + * loudly instead of writing on. Different hostname → cannot prove + * anything, treat as live. The heartbeat remains for OBSERVABILITY (the + * lock error names it so an operator can judge staleness themselves). */ private async isWriterLockStale(lock: WriterLockInfo): Promise { const os = await import('node:os') if (lock.hostname !== os.hostname()) { return false } - const heartbeatAge = Date.now() - new Date(lock.lastHeartbeat).getTime() - const pidAlive = this.isPidAlive(lock.pid) - if (!pidAlive) return true - return heartbeatAge > FileSystemStorage.WRITER_STALE_THRESHOLD_MS + return !this.isPidAlive(lock.pid) } /** diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index c3b3b3bf..b65e938e 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -612,6 +612,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { return null } + /** + * THE FENCE: verify this instance still owns its writer lock before a + * commit barrier proceeds; throw `BRAINY_WRITER_FENCED` if evicted. The + * default is a no-op — adapters without a cross-process lock model (memory, + * per-request cloud stores) have no eviction to fence against. The + * filesystem adapter overrides this; the generation store calls it at + * every flush commit and transact barrier. + */ + public async assertWriterFenceHeld(): Promise { + // No-op by default — no lock model, nothing to be evicted from. + } + /** * Start watching for cross-process flush requests. The writer Brainy * instance calls this so that out-of-process inspectors can ask for a diff --git a/tests/integration/writer-lock-fencing.test.ts b/tests/integration/writer-lock-fencing.test.ts new file mode 100644 index 00000000..86e79b99 --- /dev/null +++ b/tests/integration/writer-lock-fencing.test.ts @@ -0,0 +1,131 @@ +/** + * @module tests/integration/writer-lock-fencing + * @description The writer-lock fencing cures, from a production dev-store + * split-brain (two live writers alternating a store's id-mapper between two + * internally-consistent truths). Three laws, each pinned: + * + * 1. A LIVE writer is never auto-evicted — staleness requires PID-death. + * (The old rule evicted on heartbeat age alone, so a >60s event-loop + * stall — debugger, GC — handed the lock to a second opener while the + * first kept writing.) + * 2. A DEAD writer's lock still self-clears with narration (venue's ask). + * 3. THE FENCE: an evicted writer (force-takeover or removed lock) fails + * LOUDLY at its next commit barrier — typed BRAINY_WRITER_FENCED — and + * never advances the store. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +function lockPath(dir: string): string { + return join(dir, 'locks', '_writer.lock') +} + +async function fsBrain(dir: string): Promise { + const brain = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false + }) + await brain.init() + brains.push(brain) + return brain +} + +describe('writer-lock fencing', () => { + it('a LIVE writer with an ancient heartbeat is NOT evicted — the second opener refuses typed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-live-')) + dirs.push(dir) + await fsBrain(dir) + + // Manufacture the trigger shape: a DIFFERENT process's lock (pid 1 — + // always alive, never ours, EPERM proves liveness) with a >60s-old + // heartbeat — the blocked-event-loop costume that used to get evicted. + const lp = lockPath(dir) + const lock = JSON.parse(fs.readFileSync(lp, 'utf-8')) + lock.pid = 1 + lock.lastHeartbeat = new Date(Date.now() - 10 * 60_000).toISOString() + fs.writeFileSync(lp, JSON.stringify(lock)) + + // Old rule: heartbeat-age eviction → silent takeover → split brain. + // New rule: live PID = live writer; the second opener throws typed. + const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) + await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' }) + }, 120000) + + it("a DEAD writer's lock self-clears and the new opener proceeds", async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-dead-')) + dirs.push(dir) + const first = await fsBrain(dir) + await first.close() + brains.pop() + + // Manufacture a crashed holder: a lock naming a PID that cannot exist. + fs.mkdirSync(join(dir, 'locks'), { recursive: true }) + fs.writeFileSync( + lockPath(dir), + JSON.stringify({ + pid: 2 ** 22 + 12345, // beyond pid_max on any default Linux + hostname: os.hostname(), + startedAt: new Date().toISOString(), + lastHeartbeat: new Date().toISOString(), + version: 'test', + rootDir: dir + }) + ) + const brain = await fsBrain(dir) // must not throw + const id = await brain.add({ data: 'post-takeover write', type: NounType.Document, metadata: {} }) + expect(await brain.get(id)).not.toBeNull() + }, 120000) + + it('THE FENCE: a forced-out writer fails its next flush typed and advances nothing', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-evict-')) + dirs.push(dir) + const victim = await fsBrain(dir) + await victim.add({ data: 'pre-eviction write', type: NounType.Document, metadata: { n: 1 } }) + await victim.flush() + const genBefore = victim.generation() + + // A successor takes the lock behind the victim's back (the force-takeover + // shape: different pid + startedAt). + fs.writeFileSync( + lockPath(dir), + JSON.stringify({ + pid: process.pid + 1, + hostname: os.hostname(), + startedAt: new Date(Date.now() + 1).toISOString(), + lastHeartbeat: new Date().toISOString(), + version: 'test-successor', + rootDir: dir + }) + ) + + // The victim's next commit barrier must refuse, typed — never write on. + await victim.add({ data: 'post-eviction write', type: NounType.Document, metadata: { n: 2 } }) + await expect(victim.flush()).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' }) + expect(victim.generation(), 'committed watermark never advanced past the fence') + .toBeGreaterThanOrEqual(genBefore) + + // Transact leg: the barrier fences there too, and rolls back cleanly. + await expect( + victim.transact([ + { op: 'add', id: '00000000-0000-7000-8000-0000000fence', type: NounType.Document, data: 'fenced', metadata: {} } + ]) + ).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' }) + + // Silence the fenced instance's close-time release (it no longer owns the lock). + brains.pop() + await victim.close().catch(() => {}) + }, 120000) +}) From 314e0e6c299e629db3191f8921e5dd6e23a56a28 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 09:36:21 -0700 Subject: [PATCH 013/102] =?UTF-8?q?test(budgets):=20iron-honest=20wall-clo?= =?UTF-8?q?ck=20budgets=20=E2=80=94=203x=20the=20worst=20honest-iron=20mea?= =?UTF-8?q?surement?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven micro-budget tests were calibrated on one fast desktop and failed on other honest iron with zero functional failures (bisect-proven pre-existing; David-waived for 10.1/10.2 with this recalibration filed as the cure). Every budget is now at least 3x the worst measurement observed across three machines, each with a comment naming its calibration basis; the find-unified micro-comparison of two sub-millisecond timings becomes a ratio assertion (absolute equality of microsecond pairs can never be stable). The inference-bound trim-history correctness test gets a timeout covering its slowest observed run (174s) — its assertions are exact and untouched. These remain order-of-magnitude guards; real perf enforcement lives in the dedicated perf lanes with iron-specific budgets, per the gate-speed standard. Known non-test artifact, documented not hidden: on slow-inference machines a minutes-long awaited-embed loop can trip vitest's worker-RPC 60s tolerance ('Timeout calling onTaskUpdate') — all tests pass, vitest exits 1 on the unhandled orchestration error. The CI lanes on faster iron exit clean; if a lane ever trips it, the test moves to deterministic embeddings (its assertions are size-bookkeeping, not embedding quality). --- .../integration/find-unified-integration.test.ts | 10 ++++++++-- tests/integration/remaining-apis.test.ts | 4 +++- tests/unit/brainy/add.test.ts | 4 +++- tests/unit/brainy/batch-operations.test.ts | 15 ++++++++++++--- tests/unit/brainy/find.test.ts | 8 +++++--- .../unit/neural/NaturalLanguageProcessor.test.ts | 12 ++++++++---- tests/unit/neural/signals/EmbeddingSignal.test.ts | 10 +++++++--- 7 files changed, 46 insertions(+), 17 deletions(-) diff --git a/tests/integration/find-unified-integration.test.ts b/tests/integration/find-unified-integration.test.ts index 4370295e..94053d55 100644 --- a/tests/integration/find-unified-integration.test.ts +++ b/tests/integration/find-unified-integration.test.ts @@ -709,8 +709,14 @@ describe('Unified Find() Integration Tests', () => { expect(simpleResult.length).toBeGreaterThan(0) expect(complexResult.length).toBeGreaterThan(0) - // Simple queries should be faster - expect(simpleDuration).toBeLessThanOrEqual(complexDuration) + // These are both sub-millisecond operations on tiny fixture data, so + // comparing two microsecond-scale timings for absolute equality-class + // ordering (simple <= complex) can never be stable — timer + // resolution and scheduling noise dominate the signal. Assert only + // the order-of-magnitude property: the simple path isn't + // dramatically slower than the complex one. The +5ms floor absorbs + // noise when complexDuration itself rounds to ~0. + expect(simpleDuration).toBeLessThanOrEqual(complexDuration * 3 + 5) }) it('should use fast paths for single search types', async () => { diff --git a/tests/integration/remaining-apis.test.ts b/tests/integration/remaining-apis.test.ts index 12d60983..f7cd17e9 100644 --- a/tests/integration/remaining-apis.test.ts +++ b/tests/integration/remaining-apis.test.ts @@ -367,7 +367,9 @@ Gadget,20` const time = Date.now() - start expect(entries.length).toBe(20) - expect(time).toBeLessThan(5000) // < 5 seconds + // order-of-magnitude guard: worst honest-iron measurement 8.85s + // (32-core CPU-only box), 3x headroom + expect(time).toBeLessThan(30000) console.log(` ✅ Created and copied 20 files in ${time}ms`) }) }) diff --git a/tests/unit/brainy/add.test.ts b/tests/unit/brainy/add.test.ts index c017862c..10690e2f 100644 --- a/tests/unit/brainy/add.test.ts +++ b/tests/unit/brainy/add.test.ts @@ -452,9 +452,11 @@ describe('Brainy.add()', () => { }) // Act & Assert + // order-of-magnitude guard: worst honest-iron measurement 105ms + // (5% over the old 100ms budget), 3x headroom on the overage class await assertCompletesWithin( () => brain.add(params), - 100, // Should complete within 100ms + 300, 'Add operation' ) }) diff --git a/tests/unit/brainy/batch-operations.test.ts b/tests/unit/brainy/batch-operations.test.ts index 58b25744..889127ee 100644 --- a/tests/unit/brainy/batch-operations.test.ts +++ b/tests/unit/brainy/batch-operations.test.ts @@ -456,7 +456,9 @@ describe('Brainy Batch Operations', () => { // Verify batch operation completed successfully // Note: Performance can vary based on system load and embedding generation expect(batchIds).toHaveLength(itemCount) - expect(batchTime).toBeLessThan(5000) // Reasonable timeout for 50 items + // order-of-magnitude guard: worst honest-iron measurement 11.9s (CPU-only + // inference, 32-core box), 3x headroom for 50-item batch + expect(batchTime).toBeLessThan(40000) console.log(`Individual: ${individualTime}ms, Batch: ${batchTime}ms`) if (batchTime < individualTime) { @@ -510,7 +512,9 @@ describe('Brainy Batch Operations', () => { const totalTime = Date.now() - startTime - expect(totalTime).toBeLessThan(3000) // v5.4.0: Type-first storage takes longer + // order-of-magnitude guard: worst honest-iron measurement 6652ms + // (mixed batch under CPU-only inference), 3x headroom + expect(totalTime).toBeLessThan(20000) // Verify final state const remaining = await brain.get(initialIds[0]) @@ -556,7 +560,12 @@ describe('Brainy Batch Operations', () => { // Might throw if there's a limit expect(error).toBeDefined() } - }, 60000) + // order-of-magnitude guard: this test batches 20x the item count of the + // sibling "perform better" test above (worst measured 11.9s for 50 + // items on CPU-only honest iron); the prior 60s timeout was itself + // observed being hit, so this is 3x that floor rather than a scaled + // extrapolation, to leave real headroom for run-to-run variance + }, 180000) it('should provide meaningful error messages', async () => { try { diff --git a/tests/unit/brainy/find.test.ts b/tests/unit/brainy/find.test.ts index c9b36e64..5bead272 100644 --- a/tests/unit/brainy/find.test.ts +++ b/tests/unit/brainy/find.test.ts @@ -375,11 +375,13 @@ describe('Brainy.find()', () => { limit: 10 }) const duration = Date.now() - start - + // Assert - expect(duration).toBeLessThan(100) + // order-of-magnitude guard: worst honest-iron measurement 106ms + // (6% over the old 100ms budget), 3x headroom on the overage class + expect(duration).toBeLessThan(300) }) - + it('should handle large result sets efficiently', async () => { // Arrange - Add many entities await Promise.all( diff --git a/tests/unit/neural/NaturalLanguageProcessor.test.ts b/tests/unit/neural/NaturalLanguageProcessor.test.ts index 0800601e..79cf9b6e 100644 --- a/tests/unit/neural/NaturalLanguageProcessor.test.ts +++ b/tests/unit/neural/NaturalLanguageProcessor.test.ts @@ -343,9 +343,11 @@ describe('NaturalLanguageProcessor', () => { const duration = Date.now() - startTime expect(result).toBeDefined() - expect(duration).toBeLessThan(200) // Should be fast + // order-of-magnitude guard: worst honest-iron measurement 4.8s + // (CPU-only inference path, 32-core box); 15s budget covers 3x that + expect(duration).toBeLessThan(15000) }) - + it('should handle multiple queries efficiently', async () => { const queries = Array(10).fill('Find AI research') @@ -356,8 +358,10 @@ describe('NaturalLanguageProcessor', () => { const duration = Date.now() - startTime expect(results).toHaveLength(10) - expect(duration).toBeLessThan(2000) // Should handle batch in reasonable time - }) + // order-of-magnitude guard: worst honest-iron measurement 48.2s for 10 + // concurrent inference-path queries (CPU-only, 32-core box); ~3x headroom + expect(duration).toBeLessThan(150000) + }, 200000) it('should cache pattern matching for performance', async () => { const query = 'Find machine learning papers' diff --git a/tests/unit/neural/signals/EmbeddingSignal.test.ts b/tests/unit/neural/signals/EmbeddingSignal.test.ts index f1ff5beb..54d34b64 100644 --- a/tests/unit/neural/signals/EmbeddingSignal.test.ts +++ b/tests/unit/neural/signals/EmbeddingSignal.test.ts @@ -218,7 +218,10 @@ describe('EmbeddingSignal', () => { const finalStats = signal.getStats() expect(finalStats.historySize).toBeLessThanOrEqual(1000) // MAX_HISTORY = 1000 - }) + // Inference-bound correctness test (hundreds of real embeds): measured + // 116-174s on honest CPU-only iron across three machines — the timeout + // covers the slowest observed with headroom; the assertions are exact. + }, 600000) it('should clear history', async () => { const vector = await brain.embed('Test') @@ -577,8 +580,9 @@ describe('EmbeddingSignal', () => { const endTime = Date.now() const totalTime = endTime - startTime - // Should be reasonably fast (< 5 seconds for 100 entities) - expect(totalTime).toBeLessThan(5000) + // order-of-magnitude guard: worst honest-iron measurement 22.3s + // (CPU-only inference, 32-core box) for 100 entities, 3x headroom + expect(totalTime).toBeLessThan(70000) const stats = signal.getStats() expect(stats.calls).toBe(100) From 0991cf28e47828cb049ecb4c32fa4c69b50bdb92 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 10:11:30 -0700 Subject: [PATCH 014/102] =?UTF-8?q?fix(locks):=20the=20fence=20keys=20owne?= =?UTF-8?q?rship=20on=20pid+hostname=20=E2=80=94=20a=20same-process=20re-o?= =?UTF-8?q?pen=20never=20fences=20its=20predecessor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plant's integration lane caught it twice: the fence's startedAt-strict comparison turned the documented same-process warn-and-take-over path (two instances in one Node process — the server-restart test pattern, and the shared-default-store pattern across test files) into a flush-killer: the first instance's background flushes latched dead while its own process held the lock ('PID N no longer holds the lock — it is now held by PID N'). Ownership is per-process: pid + hostname. startedAt stays in the lock for observability but not in the fence — it protects nothing (a pid-recycled successor's victim is a dead process that runs no fence checks) and it convicted the innocent. Pinned: a same-process re-open leaves both instances' flushes working; the cross-process eviction pins unchanged. Verified under the lane's exact command: 102/102 files, 850 passed, exit 0. --- src/storage/adapters/fileSystemStorage.ts | 11 +++++++++-- tests/integration/writer-lock-fencing.test.ts | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 8fb2d2f1..b8e2a9af 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -1998,11 +1998,18 @@ export class FileSystemStorage extends BaseStorage { public override async assertWriterFenceHeld(): Promise { if (!this.writerLockInfo) return const current = await this.readWriterLock() + // Ownership is PER-PROCESS: pid + hostname, deliberately NOT startedAt. + // The documented same-process re-open path ("warn and take over" — two + // instances in one Node process, the server-restart test pattern) + // rewrites the lock with a fresh startedAt; fencing the first instance + // on that mismatch latched its background flushes dead while its own + // process held the lock (caught by the plant's integration lane, twice). + // startedAt adds nothing against pid recycling either: a recycled pid's + // victim is a DEAD process — it runs no fence checks. if ( current && current.pid === this.writerLockInfo.pid && - current.hostname === this.writerLockInfo.hostname && - current.startedAt === this.writerLockInfo.startedAt + current.hostname === this.writerLockInfo.hostname ) { return } diff --git a/tests/integration/writer-lock-fencing.test.ts b/tests/integration/writer-lock-fencing.test.ts index 86e79b99..e9f98dac 100644 --- a/tests/integration/writer-lock-fencing.test.ts +++ b/tests/integration/writer-lock-fencing.test.ts @@ -89,6 +89,23 @@ describe('writer-lock fencing', () => { expect(await brain.get(id)).not.toBeNull() }, 120000) + it('the fence does NOT fire on a same-process re-open — the documented warn-and-take-over contract stays benign', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-samepid-')) + dirs.push(dir) + const first = await fsBrain(dir) + await first.add({ data: 'first instance write', type: NounType.Document, metadata: { n: 1 } }) + + // A second instance in the SAME process takes the lock over (fresh + // startedAt) — the pattern server-restart tests use. The first + // instance's background flushes must keep working: same pid + same + // hostname IS ownership. (The plant's integration lane caught the + // startedAt-strict fence latching exactly this shape dead.) + const second = await fsBrain(dir) + await second.add({ data: 'second instance write', type: NounType.Document, metadata: { n: 2 } }) + await expect(first.flush()).resolves.toBeUndefined() + await expect(second.flush()).resolves.toBeUndefined() + }, 120000) + it('THE FENCE: a forced-out writer fails its next flush typed and advances nothing', async () => { const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-evict-')) dirs.push(dir) From 97d7564900a0b328542a7b902f30f0e6ef32b250 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 10:43:00 -0700 Subject: [PATCH 015/102] =?UTF-8?q?docs(releases):=20the=2010.3.0=20consum?= =?UTF-8?q?er=20entry=20=E2=80=94=20the=20trust-and-provenance=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 602b84e4..49876f5a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,41 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.3.0 — 2026-08-18 (the trust-and-provenance release) + +Four consumer-driven cures. Pairs with the same native accelerator line +(>=4.1.0); adopt alongside the accelerator's 4.2.0 for its paired fixes. + +- **Writer-lock fencing.** A live writer is never auto-evicted (staleness now + requires the holding process to be dead — a >60s stall is a slow writer, not + a dead one); the lock claim is atomic (no empty-file window a racer can + misread as torn); and every flush commit and transact barrier verifies lock + ownership first, so a forced-out or lock-deleted writer fails typed + (`BRAINY_WRITER_FENCED`) instead of writing on unaware — the split-brain + class a shared dev store hit is dead at all three roots. The documented + same-process re-open ("warn and take over") stays benign: ownership is + per-process. Consumers that raised stop-timeouts as mitigation can retire + them. +- **Transaction-log provenance.** `TxLogEntry` gains an optional `origin` + field — absent means a user write (existing consumers unchanged); + engine-originated commits stamp themselves (`system:embed-landing`, + `system:adoption-backfill`, `system:reconcile`), and the same stamp rides + the commit fact's meta. Activity feeds filter on fact instead of guessing; + a reported "double tick" (the deferred vector landing indistinguishable from + a user save) is cured without collapsing genuine rapid saves. +- **The attested reconcile door.** `reconcileLogDivergence(id, {attest})` + resolves the one adoption-refusing divergence class + (`log-live-canonical-absent`) with a human's word: `'deleted'` mints the + tombstone the log always lacked; `'restore'` folds the log's only copy back + into canonical; wrong-class calls refuse typed with nothing written. Loud, + narrated, single-row. +- **Iron-honest test budgets.** The wall-clock micro-budgets are recalibrated + as order-of-magnitude guards (3x the worst measurement across three machine + classes) so honest hardware differences can never again read as failures; + real performance enforcement lives in the dedicated perf lanes. + +--- + ## v10.2.0 — 2026-08-17 (adoption completes in one call) One fix, headline-sized for large stores. Pairs with the same native accelerator From 8fb6cb7e5468a3de50784a390686241c226328a9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 10:43:27 -0700 Subject: [PATCH 016/102] chore(release): 10.3.0 --- CHANGELOG.md | 9 +++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e91a6ad..5ee1e723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18) + +- docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) +- fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) +- test(budgets): iron-honest wall-clock budgets — 3x the worst honest-iron measurement (314e0e6c) +- fix(locks): live writers are never auto-evicted; evicted writers are fenced at every commit barrier (292e7c04) +- feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) + + ### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17) - docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) diff --git a/package-lock.json b/package-lock.json index e8c238f5..a5913ac7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.2.0", + "version": "10.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.2.0", + "version": "10.3.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index a366f42f..241f23a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.2.0", + "version": "10.3.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From ed7d1db97e964ad25c2b8b37afdd5bfa209a5665 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 12:53:50 -0700 Subject: [PATCH 017/102] fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production brain's first process boot after a live authority flip looked hung and was restarted three times mid-recovery — three defects with one scene. (1) THE FOLD MATERIALIZED THE LOG: peekFactsAbove(0) decoded every fact into one array (GBs of after-images on a ~7k-fact log, a GC storm, a starved write lane). The fold now STREAMS one segment-batch at a time — memory is one segment at any log size — with structural ordering asserted loudly. (2) THE FOLD WAS SILENT UNTIL DONE: minutes of boot work with zero narration is what invited the restarts. It now announces itself BEFORE the work ('do not restart, the fold is finite') and prints progress every thousand facts. (3) THE CHAIN COULD ONLY ARM AT A CRASH: a live mid-session flip left the fold checkpoint unfounded, so the brain's first unclean boot paid a whole-log fold. Adoption now founds the checkpoint AT THE FLIP — one paged full canonical barrier (bounded memory), then the stamp — so bounded recovery holds from minute zero for every store that flips, at any size. Pinned: a non-fresh flip stamps immediately; the first post-flip unclean boot folds bounded (an unflushed at-ack fact above the checkpoint is restored; a barrier-covered row below it is outside the fold). Kill matrix and both adoption suites green alongside. --- src/brainy.ts | 51 +++++++ src/db/factLog.ts | 37 +++++ src/db/generationStore.ts | 128 +++++++++++++----- .../integration/fold-checkpoint-bound.test.ts | 32 +++++ 4 files changed, 215 insertions(+), 33 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index d1ec144b..fb1c1614 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8392,6 +8392,57 @@ export class Brainy implements BrainyInterface { // Fold-checkpoint chain, phase 2: the flip is recorded — open the stamp // gate so the next flush/close barrier writes the first checkpoint. this.generationStore.completeFoldCheckpointBootstrap() + // ARM-AT-FLIP for the NON-FRESH brain (the chain refused the fresh-brain + // arm because committed > 0): run one paged FULL canonical barrier now — + // every live row's canonical bytes fsynced, bounded memory — then stamp + // the first checkpoint. Without this, the chain could only arm at the + // brain's first crash, and that crash paid a WHOLE-LOG fold: a production + // brain hit exactly that on its first post-flip boot (a full-log + // materializing fold, restarted three times mid-flight). Adoption already + // pays O(N) oracle work; one more O(N) barrier founds bounded recovery + // from minute zero. + if (!this.generationStore.foldCheckpointChainArmed()) { + const PAGE = 500 + let synced = 0 + prodLog.info( + `[Brainy] adoptLogAuthority: founding the fold checkpoint — syncing every ` + + `row's canonical bytes (paged; progress every 2000 rows)` + ) + let offset = 0 + let cursor: string | undefined + for (;;) { + const page = await this.storage.getNouns({ + pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } + }) + const ids = page.items.map((i) => (i as { id: string }).id) + if (ids.length > 0) { + await this.storage.syncEntityCanonical?.(ids, []) + synced += ids.length + if (synced % 2000 < PAGE && synced >= 2000) { + prodLog.info(`[Brainy] adoptLogAuthority: checkpoint founding — ${synced} rows synced`) + } + } + if (page.hasMore && page.nextCursor) { cursor = page.nextCursor; offset += ids.length; continue } + if (page.hasMore && !page.nextCursor) { offset += PAGE; continue } + break + } + let vOffset = 0 + let vCursor: string | undefined + for (;;) { + const page = await this.storage.getVerbs({ + pagination: vCursor ? { limit: PAGE, cursor: vCursor } : { limit: PAGE, offset: vOffset } + }) + const ids = page.items.map((i) => (i as { id: string }).id) + if (ids.length > 0) { + await this.storage.syncEntityCanonical?.([], ids) + synced += ids.length + } + if (page.hasMore && page.nextCursor) { vCursor = page.nextCursor; vOffset += ids.length; continue } + if (page.hasMore && !page.nextCursor) { vOffset += PAGE; continue } + break + } + await this.generationStore.stampFoldCheckpointAfterFullBarrier() + } return report } diff --git a/src/db/factLog.ts b/src/db/factLog.ts index 82949fb6..ca130454 100644 --- a/src/db/factLog.ts +++ b/src/db/factLog.ts @@ -770,6 +770,43 @@ export class FactLog { * segments directly; the torn tail's invalid suffix is ignored exactly * like open() would). */ + /** + * STREAMING twin of {@link FactLog.peekFactsAbove} for the recovery fold: + * yields facts above the bound one SEGMENT at a time, ascending, without + * ever materializing the whole log (a production first-boot fold OOM-class + * allocation storm came from exactly that — GBs of decoded after-images in + * one array while the process looked hung). Memory is one segment's worth. + * Works manifest-direct (safe before {@link FactLog.open}). Ordering is + * structural (segments rotate in order; appends are ordered within one) and + * ASSERTED — a violation aborts loudly, never a silent misordered replay. + */ + async *streamFactsAbove(committedGeneration: number): AsyncGenerator { + const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null + if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return + if (stored.formatVersion !== FACTS_FORMAT_VERSION) return + const files = [...stored.segments.map((s) => s.file)] + if (stored.tailSegment) files.push(stored.tailSegment) + let lastGen = committedGeneration + for (const file of files) { + const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) + if (bytes === null) continue + const { facts } = parseSegment(file, bytes) + const batch: CommitFact[] = [] + for (const f of facts) { + if (f.generation <= committedGeneration) continue + if (f.generation <= lastGen) { + throw new Error( + `fact log: streamFactsAbove found non-ascending generations ` + + `(${f.generation} after ${lastGen} in ${file}) — refusing to replay out of order` + ) + } + lastGen = f.generation + batch.push(f) + } + if (batch.length > 0) yield batch + } + } + async peekFactsAbove(committedGeneration: number): Promise { const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return [] diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 065b3659..bfb68959 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -637,33 +637,63 @@ export class GenerationStore { this.foldCheckpointChainValid = checkpoint !== null || this.committed === 0 this.foldCheckpoint = foldBound if (uncleanOpen) this.foldCheckpointChainValid = true - const factsToReplay = uncleanOpen - ? await this.factLog.peekFactsAbove(foldBound) - : orphans - if (factsToReplay.length > 0) { - let replayed = 0 - for (const fact of factsToReplay) { - for (const op of fact.ops) { - const image = - op.record === null - ? { metadata: null, vector: null } - : { metadata: op.record.metadata, vector: op.record.vector } - if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) - else await this.storage.writeNounRaw(op.id, image) - this.noteCheckpointDirty(op.kind, op.id) - } - replayed++ - if (fact.generation > this.committed) { - this.committed = fact.generation - this.appendCommittedGen(fact.generation) - this.setDelta(fact.generation, { - nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)), - verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)), - timestamp: fact.timestamp, - bytes: 0 - }) - } + // THE FOLD STREAMS AND NARRATES. A production first boot after a live + // flip folded ~7k facts by materializing them all (GBs of decoded + // after-images, a GC storm, a starved write lane) in SILENCE — the + // operator restarted the process three times mid-fold, each restart + // making the next boot unclean again. Two laws from that day: the + // fold consumes the log one segment-batch at a time (memory = one + // segment, any log size), and it announces itself BEFORE the work + // with progress lines DURING it — an operator who can see a fold + // converging lets it finish. + const foldKind = uncleanOpen + ? foldBound > 0 + ? `BOUNDED fold above checkpoint ${foldBound}` + : 'WHOLE-LOG fold' + : 'above-manifest replay' + let replayed = 0 + const replayFact = async (fact: CommitFact): Promise => { + for (const op of fact.ops) { + const image = + op.record === null + ? { metadata: null, vector: null } + : { metadata: op.record.metadata, vector: op.record.vector } + if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) + else await this.storage.writeNounRaw(op.id, image) + this.noteCheckpointDirty(op.kind, op.id) } + replayed++ + if (replayed % 1000 === 0) { + prodLog.warn( + `[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` + + `(at generation ${fact.generation}); do not restart, the fold is finite` + ) + } + if (fact.generation > this.committed) { + this.committed = fact.generation + this.appendCommittedGen(fact.generation) + this.setDelta(fact.generation, { + nouns: new Set(fact.ops.filter((o) => o.kind === 'noun').map((o) => o.id)), + verbs: new Set(fact.ops.filter((o) => o.kind === 'verb').map((o) => o.id)), + timestamp: fact.timestamp, + bytes: 0 + }) + } + } + if (uncleanOpen) { + prodLog.warn( + `[GenerationStore] log-authority recovery: ${foldKind} beginning ` + + `(unclean shutdown detected) — streaming replay, bounded memory, ` + + `progress every 1000 facts. Do not restart the process; a restart ` + + `re-pays the whole fold.` + ) + for await (const batch of this.factLog.streamFactsAbove(foldBound)) { + for (const fact of batch) await replayFact(fact) + } + } else { + for (const fact of orphans) await replayFact(fact) + } + if (replayed > 0) { if (this.counter < this.committed) this.counter = this.committed await this.persistCounterUnlocked() const manifest: GenerationManifest = { @@ -676,13 +706,7 @@ export class GenerationStore { await this.storage.syncRawObjects([MANIFEST_PATH]) prodLog.warn( `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + - `canonical (${ - uncleanOpen - ? foldBound > 0 - ? `BOUNDED fold above checkpoint ${foldBound} — unclean shutdown` - : 'WHOLE-LOG fold — unclean shutdown' - : 'above-manifest' - }; committed at ${this.committed}) — an acked write is never lost` + `canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost` ) } // A recovery fold re-applied (and the barrier below re-syncs) every @@ -897,6 +921,44 @@ export class GenerationStore { this.authorityIsLog = true } + /** Whether the fold-checkpoint chain is armed (a bounded fold is possible). */ + foldCheckpointChainArmed(): boolean { + return this.foldCheckpointChainValid + } + + /** + * @description Stamp the fold checkpoint after the caller has completed a + * FULL canonical barrier (every live row's canonical bytes fsynced, paged — + * the adoption path does this right after a non-fresh flip). The stamp + * asserts total coverage, so it may ONLY be called when the barrier walked + * everything; stamp-after-data is the caller's ordering to keep. Arms the + * chain: the brain's first unclean boot folds (checkpoint, head] instead of + * the whole log — a production first boot after a live flip paid a full-log + * fold through three mid-fold restarts because the chain could previously + * only arm at a crash. + */ + async stampFoldCheckpointAfterFullBarrier(): Promise { + return this.withMutex(async () => { + if (!this.authorityIsLog || !this.factLog) { + throw new Error( + 'stampFoldCheckpointAfterFullBarrier: only a log-authority brain stamps a fold checkpoint' + ) + } + this.foldCheckpointChainValid = true + // The full barrier supersedes any accumulated partial set. + this.checkpointDirtyNouns = new Set() + this.checkpointDirtyVerbs = new Set() + const target = this.committed + await this.storage.writeRawObject(FOLD_CHECKPOINT_PATH, { generation: target }) + await this.storage.syncRawObjects([FOLD_CHECKPOINT_PATH]) + this.foldCheckpoint = target + prodLog.info( + `[GenerationStore] fold checkpoint founded at generation ${target} — ` + + `crash recovery is bounded from this moment` + ) + }) + } + /** * @description Adoption-time chain bootstrap, abort — called when an * adoption attempt throws or refuses after phase 1. Disarms the chain and diff --git a/tests/integration/fold-checkpoint-bound.test.ts b/tests/integration/fold-checkpoint-bound.test.ts index 60bcce5e..2f21248b 100644 --- a/tests/integration/fold-checkpoint-bound.test.ts +++ b/tests/integration/fold-checkpoint-bound.test.ts @@ -161,6 +161,38 @@ describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], nev expect(stamped, 'the first whole-log fold is the chain’s base case — it stamps').toBe(committedOf(reopened)) }, 120000) + it('ARM-AT-FLIP: a non-fresh adoption founds the checkpoint immediately — the first post-flip boot folds BOUNDED, never whole-log', async () => { + const dir = trackDir() + // The production shape: a brain with history flips LIVE (no crash ever). + const brain = await openBrain(dir, { logAuthority: 'defer' }) + liveBrains.push(brain) + const preFlip = await brain.add({ data: 'pre-flip resident', type: NounType.Document, metadata: { era: 'tree' } }) + await brain.flush() + expect(readCheckpoint(dir), 'no checkpoint before the flip').toBeNull() + + const report = await brain.adoptLogAuthority() + expect(report.verdict).toBe('green') + // THE PIN: the flip itself founded the checkpoint — no crash required. + const founded = readCheckpoint(dir) + expect(founded, 'checkpoint founded at flip').toBe(committedOf(brain)) + + // First post-flip boot, unclean (the production first-restart shape): + // a post-flip write above the checkpoint is restored FROM ITS AT-ACK FACT + // (deliberately NOT flushed — a flush would barrier-sync it and advance + // the stamp over it, making its loss synthetic); the pre-flip row (its + // baseline fact ≤ checkpoint, its bytes barrier-synced at the flip) is + // OUTSIDE the fold — vaporizing it synthetically proves the bound. + const postFlip = await brain.add({ data: 'post-flip write', type: NounType.Document, metadata: { era: 'log' } }) + await abandonAsCrashed(liveBrains.pop()!) + dropCanonicalNoun(dir, preFlip) + dropCanonicalNoun(dir, postFlip) + + const reopened = await openBrain(dir, { logAuthority: 'adopt' }) + liveBrains.push(reopened) + expect(await reopened.get(postFlip), 'above-checkpoint fact re-applied').not.toBeNull() + expect(await reopened.get(preFlip), 'below-checkpoint fact skipped — the fold is bounded on the FIRST post-flip boot').toBeNull() + }, 240000) + it('a tree-authority brain never stamps a checkpoint', async () => { const dir = trackDir() const brain = await openBrain(dir, { logAuthority: 'defer' }) From 900cc89564275e9647d8ea45cb099a2e24b308ff Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 13:18:55 -0700 Subject: [PATCH 018/102] =?UTF-8?q?docs(releases):=20the=2010.3.1=20consum?= =?UTF-8?q?er=20entry=20=E2=80=94=20the=20fold=20that=20behaves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 49876f5a..cc0272c3 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,33 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.3.1 — 2026-08-18 (the fold that behaves) + +Three recovery cures from one production first-boot incident (a brain's first +process restart after a live storage-authority flip looked hung and was +restarted three times mid-recovery). **Adopt this version before flipping +brains with existing history** — it is the intended adoption target for +fleets moving to the crash-safe authority. + +- **Recovery streams.** The boot-time log fold now consumes the generation + log one segment-batch at a time — memory stays bounded at one segment for + any log size. Previously it materialized every fact into one array, which + on a ~7k-fact log produced multi-GB allocation pressure and a process that + looked wedged while it worked. +- **Recovery narrates.** The fold announces itself before the work begins + ("recovery fold beginning — do not restart, the fold is finite") and prints + progress every thousand facts. A visible fold gets to finish; a silent one + gets killed by a well-meaning operator, and each kill makes the next boot + pay the whole fold again. +- **Bounded recovery from the flip itself.** Adopting the log authority now + founds the recovery checkpoint at the moment of the flip (one paged + canonical sync, bounded memory, then the stamp) — so even the FIRST unclean + shutdown after a flip replays only the log's tail. Previously the bound + could only establish itself at a completed crash recovery, which is exactly + the recovery the incident kept interrupting. + +--- + ## v10.3.0 — 2026-08-18 (the trust-and-provenance release) Four consumer-driven cures. Pairs with the same native accelerator line From 522b0cf827489f91b3cf91f95eb0af7cae6d5ae7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 18 Aug 2026 13:19:17 -0700 Subject: [PATCH 019/102] chore(release): 10.3.1 --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee1e723..f99584e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18) + +- docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) +- fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) + + ### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18) - docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) diff --git a/package-lock.json b/package-lock.json index a5913ac7..afce417d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.3.0", + "version": "10.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.3.0", + "version": "10.3.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 241f23a8..75e5bfbc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.3.0", + "version": "10.3.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 1e046aa115637b9b0971b8fe301152e318ed5bc8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 20 Aug 2026 08:22:48 -0700 Subject: [PATCH 020/102] ci(gate): the machine-health preflight and the truncation verdict guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two guards for every gate lane, born from the 2026-08-13 lost-day ledger. gate-preflight.sh refuses a lane on a machine that cannot be trusted to produce honest numbers — co-tenant processes named by pid and command, load average, CPU governor, disk floors — one FATAL line per violation so the operator can act from the message alone. vitest-verdict-check.sh refuses a suite log that cannot be trusted as a verdict — missing or mismatched summary counts, files that never executed (a truncated run once read as green from three files of ninety-nine), and worker-pool death signatures. Both verified live: the preflight correctly refuses this workstation naming its actual offenders; the verdict guard passes/fails five fixture shapes (clean, wrong-count, truncated, worker-death, no-summary) and both CLI modes. Wire-up into the CI lanes rides the runner program. --- scripts/gate/README.md | 85 +++++++++++ scripts/gate/gate-preflight.sh | 206 +++++++++++++++++++++++++++ scripts/gate/vitest-verdict-check.sh | 158 ++++++++++++++++++++ 3 files changed, 449 insertions(+) create mode 100644 scripts/gate/README.md create mode 100755 scripts/gate/gate-preflight.sh create mode 100755 scripts/gate/vitest-verdict-check.sh diff --git a/scripts/gate/README.md b/scripts/gate/README.md new file mode 100644 index 00000000..0a8afab0 --- /dev/null +++ b/scripts/gate/README.md @@ -0,0 +1,85 @@ +# Gate Guards + +Two standalone scripts that stand between a test/build gate and a false +verdict: one refuses to let the gate start on a noisy machine, the other +refuses to let a truncated or crashed vitest run be read as green. + +## Why these exist + +Both guards exist because of the 2026-08-13 lost-day ledger: a gate ran on +a machine under load, and separately a vitest worker pool died mid-suite +while still printing a plausible-looking summary line, and in both cases +the bad result was trusted and acted on for the better part of a day before +anyone noticed. Neither failure mode announces itself — a loaded machine +still finishes and reports numbers, and a truncated test run still prints a +`Test Files` / `Tests` line — so both guards check the evidence explicitly +rather than trusting that a gate finishing means the gate was valid. + +## gate-preflight.sh + +Run before any gate lane starts. Exits 1 the moment the machine isn't +gate-clean, with one `FATAL:` line per violation naming the exact offender +(the pid and command, the path, the measured value). Prints one `OK:` line +per check that passes. `WARNING:` lines mark checks that were skipped, not +failures. + +Checks: + +| # | Check | Default threshold | Override | +|---|-------|--------------------|----------| +| a | 1-minute load average | `nproc / 2` | `GATE_MAX_LOAD` | +| b | any non-allowlisted process over 50% of one core | 50% | `GATE_ALLOW_REGEX` (extra pattern matched against the process's args) | +| c | cpu0 scaling governor must be `performance` | — | none (warns and skips if the sysfs path is absent) | +| d | free space on `/` and `/tmp` | 10G each | `GATE_SKIP_DISK_CHECK=1` to skip entirely | + +The allowlist for check (b) is always: this script's own process tree +(its ancestors and its direct child processes), `sshd`, `systemd`, and +kernel threads (recognizable by args wrapped in brackets, e.g. +`[kworker/0:1]`). `GATE_ALLOW_REGEX` extends it — it does not replace it. + +## vitest-verdict-check.sh + +Run after every vitest lane, against that lane's captured log. Fails +loudly, quoting the exact line or string that tripped it, when the log's +own summary can't be trusted: + +- no `Test Files` (or, in `--count-tests` mode, `Tests`) summary line is + present at all +- the parenthesized total in that line doesn't match what was expected +- fewer files/tests are accounted for (passed + failed + skipped) than the + total claims — a truncated run +- the log contains `Unhandled Error` or `Timeout calling` anywhere — a dead + worker pool, regardless of what the summary line claims + +``` +vitest-verdict-check.sh +vitest-verdict-check.sh --count-tests +``` + +The first form checks `Test Files` for an exact match. The second checks +`Tests` for a minimum (a floor, not an exact count, since the total number +of individual tests moves more often than the number of test files). + +## Wiring into a CI lane + +```sh +# Before any lane that will report a verdict: +scripts/gate/gate-preflight.sh || exit 1 + +# Run the suite, capturing its output: +npx vitest run tests/unit 2>&1 | tee /tmp/unit.log + +# After every vitest lane, check the log against the actual file count: +EXPECTED_FILES=$(ls tests/unit/**/*.test.ts | wc -l) +scripts/gate/vitest-verdict-check.sh /tmp/unit.log "$EXPECTED_FILES" || exit 1 +``` + +## Exit-code contract + +| Script | Exit 0 | Exit 1 | +|--------|--------|--------| +| `gate-preflight.sh` | machine is gate-clean | one or more `FATAL:` violations printed | +| `vitest-verdict-check.sh` | log's summary is trustworthy and matches | usage error, missing/unreadable log, or one or more `FATAL:` violations printed | + +Non-zero from either script means: do not trust the gate that was about to +run, or the result of the one that just ran. diff --git a/scripts/gate/gate-preflight.sh b/scripts/gate/gate-preflight.sh new file mode 100755 index 00000000..c6208f49 --- /dev/null +++ b/scripts/gate/gate-preflight.sh @@ -0,0 +1,206 @@ +#!/bin/bash +set -euo pipefail + +# Brainy Gate Preflight +# Refuses to let a test/build gate run on a machine that isn't clean enough +# to trust the numbers it produces. See scripts/gate/README.md for why (the +# 2026-08-13 lost-day ledger). +# +# Checks: 1-minute load average, any non-allowlisted process pinning a core, +# the cpu0 scaling governor, and free space on / and /tmp. +# +# Exit 0 and print one OK line per passing check when the machine is clean. +# Exit 1 and print one FATAL line per violation, naming the offender, when +# it is not. +# +# Known trap: a helper function whose last executed statement is a `while` +# (or any command whose own exit status happens to be nonzero) hands that +# status back as the function's return value. Called as a plain statement, +# that silently kills this script under `set -e`. Every helper below ends +# on an explicit `return 0` as its own statement, never on a loop or test. +# +# The same failure mode hides in plainer-looking lines too: `var=$(cmd)` is +# a bare assignment, so `set -e` DOES treat a nonzero `cmd` (or, under +# `pipefail`, a nonzero stage anywhere in `cmd`'s pipeline) as a failure of +# that statement and kills the script right there — even mid-loop, even +# when the "failure" is routine (a process that exited before a second +# lookup, a path that doesn't exist). Every such assignment below is paired +# with an explicit `|| var=""` fallback so a routine miss degrades to an +# empty value instead of an exit. + +VIOLATIONS=0 +ANCESTOR_PIDS="" + +fatal() { + echo "FATAL: $1" + VIOLATIONS=$((VIOLATIONS + 1)) +} + +ok() { + echo "OK: $1" +} + +# Walks this process's parent chain up to pid 1, then takes one snapshot of +# its direct children (the ps/read pipeline in check_processes), and +# records both in ANCESTOR_PIDS — so the process-scan below can recognize +# its own tree (the shell/terminal/session that launched it, plus its own +# helper commands) instead of flagging it. Children are captured once, up +# front, rather than re-queried per row later, so a helper command that has +# already exited by the time it's looked up can't be mistaken for a miss. +build_ancestor_pids() { + local pid="$$" + local ppid child + ANCESTOR_PIDS=" $pid " + while [ "$pid" != "1" ]; do + ppid=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d ' ') || ppid="" + if [ -z "$ppid" ]; then + break + fi + ANCESTOR_PIDS="${ANCESTOR_PIDS}${ppid} " + pid="$ppid" + done + + while IFS= read -r child; do + [ -z "$child" ] && continue + ANCESTOR_PIDS="${ANCESTOR_PIDS}${child} " + done < <(ps --ppid "$$" -o pid= 2>/dev/null || true) + + return 0 +} + +# (a) 1-minute load average vs. threshold (default: nproc / 2). +check_load() { + local max_load="${GATE_MAX_LOAD:-}" + if [ -z "$max_load" ]; then + max_load=$(( $(nproc) / 2 )) + if [ "$max_load" -lt 1 ]; then + max_load=1 + fi + fi + + local load_1m + load_1m=$(cut -d' ' -f1 /proc/loadavg) + + if awk -v l="$load_1m" -v m="$max_load" 'BEGIN { exit !(l > m) }'; then + fatal "1-minute load average ${load_1m} exceeds threshold ${max_load} (GATE_MAX_LOAD=${max_load})" + else + ok "1-minute load average ${load_1m} is within threshold ${max_load}" + fi + return 0 +} + +# (b) any process outside the allowlist pinning more than half a core. +# Parsed with `read` into named fields, not an awk/cut chain — a fixed-column +# awk/cut split on `ps` output duplicated fields the first time this was +# tried, because process args vary in word count. `read` with a fixed list +# of variables dumps everything left over into the last one (args), which +# handles that correctly. +check_processes() { + local max_pcpu=50 + local extra_regex="${GATE_ALLOW_REGEX:-}" + local violation_found=0 + local line pcpu pid args pcpu_int + + while IFS= read -r line; do + [ -z "$line" ] && continue + read -r pcpu pid args <<< "$line" + + # Kernel threads report their comm in brackets, e.g. "[kworker/0:1]". + case "$args" in + \[*\]) continue ;; + esac + + # This script's own tree: its ancestors (shell, terminal, session) and + # its direct children, both captured once by build_ancestor_pids. + case " $ANCESTOR_PIDS " in + *" $pid "*) continue ;; + esac + + case "$args" in + *sshd*|*systemd*) continue ;; + esac + + if [ -n "$extra_regex" ] && [[ "$args" =~ $extra_regex ]]; then + continue + fi + + pcpu_int="${pcpu%.*}" + if [ -z "$pcpu_int" ]; then + pcpu_int=0 + fi + if [ "$pcpu_int" -gt "$max_pcpu" ]; then + fatal "pid ${pid} ('${args}') is using ${pcpu}% of one core" + violation_found=1 + fi + done < <(ps -eo pcpu,pid,args --sort=-pcpu | tail -n +2) + + if [ "$violation_found" -eq 0 ]; then + ok "no process outside the allowlist exceeds ${max_pcpu}% of one core" + fi + return 0 +} + +# (c) cpu0 scaling governor must be "performance". Skipped with a warning +# (not a violation) when the sysfs path doesn't exist on this machine. +check_governor() { + local gov_path="/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" + if [ ! -r "$gov_path" ]; then + echo "WARNING: ${gov_path} not present; skipping governor check" + return 0 + fi + + local governor + governor=$(cat "$gov_path" 2>/dev/null) || governor="" + if [ "$governor" != "performance" ]; then + fatal "cpu0 governor is '${governor}', not 'performance'" + else + ok "cpu0 governor is 'performance'" + fi + return 0 +} + +# (d) free-space floors on / and /tmp (default 10G each). Skip entirely via +# GATE_SKIP_DISK_CHECK=1. +check_disk() { + if [ "${GATE_SKIP_DISK_CHECK:-0}" = "1" ]; then + echo "WARNING: disk free-space check skipped (GATE_SKIP_DISK_CHECK=1)" + return 0 + fi + + local floor_gb=10 + local floor_bytes=$((floor_gb * 1024 * 1024 * 1024)) + local path avail_bytes avail_gb + + for path in / /tmp; do + avail_bytes=$(df --output=avail -B1 "$path" 2>/dev/null | tail -n 1 | tr -d ' ') || avail_bytes="" + if [ -z "$avail_bytes" ]; then + echo "WARNING: could not determine free space on ${path}; skipping" + continue + fi + if [ "$avail_bytes" -lt "$floor_bytes" ]; then + avail_gb=$((avail_bytes / 1024 / 1024 / 1024)) + fatal "${path} has only ${avail_gb}G free, below the ${floor_gb}G floor" + else + ok "${path} has enough free space (floor ${floor_gb}G)" + fi + done + return 0 +} + +echo "Brainy gate preflight" +echo "----------------------" + +build_ancestor_pids +check_load +check_processes +check_governor +check_disk + +echo "----------------------" +if [ "$VIOLATIONS" -gt 0 ]; then + echo "FATAL: gate preflight failed with ${VIOLATIONS} violation(s) — machine is not gate-clean" + exit 1 +fi + +echo "gate preflight passed — machine is gate-clean" +exit 0 diff --git a/scripts/gate/vitest-verdict-check.sh b/scripts/gate/vitest-verdict-check.sh new file mode 100755 index 00000000..36243a1d --- /dev/null +++ b/scripts/gate/vitest-verdict-check.sh @@ -0,0 +1,158 @@ +#!/bin/bash +set -euo pipefail + +# Brainy Vitest Verdict Check +# Confirms a vitest run's own summary line is trustworthy before anything +# downstream treats a green run as green. See scripts/gate/README.md for why +# (the 2026-08-13 lost-day ledger). +# +# Usage: +# vitest-verdict-check.sh +# vitest-verdict-check.sh --count-tests +# +# The first form checks the "Test Files" summary line's total against an +# exact expected count. The second checks the "Tests" summary line's total +# against a minimum. Both also fail on any sign the worker pool died +# mid-run, whether or not a summary line still made it into the log. +# +# Exit 0 and print one OK line per passing check when the log is clean. +# Exit 1 and print one FATAL line per violation, quoting the exact line or +# string that tripped it, when it is not. +# +# Known trap (shared with gate-preflight.sh): every helper below ends on an +# explicit `return 0` as its own statement, never on a loop or test, so a +# helper's last command can never hand its own exit status back as the +# function's under `set -e`. The same applies to `var=$(cmd)` assignments +# mid-helper: a bare assignment IS checked by `set -e`, so a `grep` that +# legitimately finds nothing (exit 1) would otherwise kill the script +# instead of just leaving the variable empty — every such assignment below +# is paired with an explicit `|| true` inside the substitution. + +usage() { + echo "Usage: $0 " + echo " $0 --count-tests " + exit 1 +} + +MODE="files" +if [ "${1:-}" = "--count-tests" ]; then + MODE="tests" + shift +fi + +LOG_FILE="${1:-}" +THRESHOLD="${2:-}" + +if [ -z "$LOG_FILE" ] || [ -z "$THRESHOLD" ]; then + usage +fi + +if [ ! -f "$LOG_FILE" ]; then + echo "FATAL: log file '${LOG_FILE}' does not exist" + exit 1 +fi + +if ! [[ "$THRESHOLD" =~ ^[0-9]+$ ]]; then + echo "FATAL: threshold '${THRESHOLD}' is not a non-negative integer" + exit 1 +fi + +VIOLATIONS=0 + +fatal() { + echo "FATAL: $1" + VIOLATIONS=$((VIOLATIONS + 1)) +} + +ok() { + echo "OK: $1" +} + +# Vitest colorizes its summary with ANSI escapes; strip them before parsing +# anything, or the color codes end up embedded in the fields we grep for. +CLEAN_LOG="$(sed 's/\x1b\[[0-9;]*m//g' "$LOG_FILE")" + +# Worker-pool death: if either string appears, the run's own summary line — +# even if present and even if its numbers look fine — cannot be trusted, +# because the process died mid-suite and vitest's own accounting is what +# died with it. +check_worker_death() { + if echo "$CLEAN_LOG" | grep -q "Unhandled Error"; then + fatal "log contains 'Unhandled Error' — worker pool died mid-run" + fi + if echo "$CLEAN_LOG" | grep -q "Timeout calling"; then + fatal "log contains 'Timeout calling' — worker pool died mid-run" + fi + return 0 +} + +# Shared shape between the "Test Files" and "Tests" summary lines: +#