From b8475cc86a9c5dca8c5c34f84f1e314e76369ef7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 13:00:42 -0700 Subject: [PATCH 01/24] =?UTF-8?q?fix(release):=20the=20release=20page=20po?= =?UTF-8?q?sts=20to=20this=20repository=20=E2=80=94=20soulcraftlabs/open-b?= =?UTF-8?q?rainy,=20never=20the=20engine's?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 11 POSTed to repos/soulcraft/brainy while printing the correct URL; dormant only because FORGEJO_RELEASE_TOKEN was unset. Found during the 10.4.4 cut verification. --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 08293e3a..67ad1053 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -237,7 +237,7 @@ fi # and RELEASES.md are the record; this just gives The Source's UI a release page). echo -e "${BLUE}๐Ÿ”Ÿ Creating release page on The Source...${NC}" if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then - if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ + if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then echo -e "${GREEN}โœ… Release page created on The Source${NC}\n" From 298cb6dacaac9ef80db65a723d65cbd53c15d23e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:07:18 -0700 Subject: [PATCH 02/24] fix(recovery): a torn generation-log tail is a terminal verdict, never a wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of one defect, found by a seeded-SIGKILL crash lane. THE FALSE POSITIVE. stampEntityTree() recorded generationStore.generation() โ€” the ALLOCATED counter, a number a write in flight has claimed and may never commit โ€” while the JSDoc beside it already said the source is the committed generation. Every crash inside a write window therefore produced a spurious verdict at the next open: either 'sourceGeneration N is ahead of the log head N-1' (the allocated generation died with the process) or 'rollup invariant nounCount: stamped X, observed Y' (the recovery fold folded facts the stamp's counts predate). Both told the operator to run repairIndex() โ€” a whole-store recount โ€” for a store that was coherent. Measured before this commit: 4 of 11 SIGKILL cycles on a healthy store raised one of the two. The stamp and the open now both read committedGeneration(), which is what every other open-time watermark in the class already reasons about. THE TERMINAL VERDICT. A stamp still ahead of committed truth after the recovery fold witnesses a generation that is not in the log โ€” the stamp's fsync outlived the tail's, and there is nothing to arrive. That is its own verdict state now ('torn'), never folded in with 'incoherent': the two have opposite cures. A writer open demotes it โ€” the unusable stamped surface is re-derived at the committed generation from the live counters, O(1), straight-line, no loop and no await on external progress, narrated with both count sets, the stamp's path and its committedAt. A read-only open cannot re-stamp, so it says so and names the cure instead of guessing, and still serves. Neither branch waits, and neither locks an owner out of a canonical tree the stamp only describes. Pins: the verifier returns the torn verdict with both generations; a fabricated head-behind-source store narrates precisely, demotes inside a bounded open, serves its rows, and is quiet at the next open (the demotion converges); a read-only open narrates the same verdict and leaves the bytes untouched. --- src/brainy.ts | 121 +++++++++++++++++++- src/db/familyStamp.ts | 32 ++++-- tests/integration/entity-tree-stamp.test.ts | 104 ++++++++++++++++- 3 files changed, 241 insertions(+), 16 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 70d46973..da04577e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -12497,6 +12497,18 @@ export class Brainy implements BrainyInterface { * healed by `repairIndex()`, whose unconditional recount rebuilds the * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write * fault warns loudly but never fails the flush that carried real data. + * + * THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is + * the ALLOCATED counter โ€” a number a write in flight has claimed and may + * never commit. Stamping it made the stamp's generation label a claim about + * counts it was not taken at, and every crash inside a write window then + * produced a spurious verdict at the next open: either `sourceGeneration N + * is ahead of the log head N-1` (the allocated generation died with the + * process) or `rollup invariant 'nounCount': stamped X, observed Y` (the + * recovery fold folded facts the stamp's counts predate). MEASURED on the + * crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on + * a coherent store raised one of those two verdicts, each of them naming + * `repairIndex()` โ€” a whole-store recount โ€” as the cure for nothing. */ private async stampEntityTree(): Promise { if (this.isReadOnly) return @@ -12507,7 +12519,7 @@ export class Brainy implements BrainyInterface { ]) await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { family: 'entity-tree', - sourceGeneration: this.generationStore.generation(), + sourceGeneration: this.generationStore.committedGeneration(), members: { mode: 'rollup', invariants: { nounCount, verbCount } } }) } catch (error) { @@ -12520,16 +12532,24 @@ export class Brainy implements BrainyInterface { /** * @description Open-time coherence check for the entity tree's family stamp: - * compare `sourceGeneration` against the log head and the stamped rollup - * invariants against the live counters. Verdicts: + * compare `sourceGeneration` against the store's COMMITTED generation and + * the stamped rollup invariants against the live counters. Verdicts: * - `coherent` / `absent` (legacy store; first flush stamps) โ†’ silent. * - `behind` โ†’ benign for the tree (it is written BY the commit; only the * stamp is stale โ€” a crash landed between commit and flush). Refreshed at * the next flush. + * - `torn` โ†’ a TORN GENERATION-LOG TAIL, handled by + * {@link demoteTornEntityTreeStamp}: terminal, never a wait. * - `incoherent` โ†’ LOUD: the tree or its counters diverged from what was * stamped โ€” `repairIndex()` recounts from canonical and re-stamps. * Never blocks open; a fault reading the stamp is surfaced as unverifiable, * never conflated with absence. + * + * THE COMPARISON IS AGAINST `committedGeneration()`, matching what + * {@link stampEntityTree} writes and what every other open-time watermark in + * this class already reasons about (the fact-scan capability, the metadata / + * graph / HNSW watermark verdicts). Comparing against the allocated counter + * was the one place that disagreed, and disagreeing was the whole defect. */ private async verifyEntityTreeStamp(): Promise { let stamp: FamilyStamp | null @@ -12546,11 +12566,16 @@ export class Brainy implements BrainyInterface { this.storage.getNounCount(), this.storage.getVerbCount() ]) - const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { + const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), { nounCount, verbCount }) - if (verdict.state === 'incoherent') { + if (verdict.state === 'torn') { + await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, { + nounCount, + verbCount + }) + } else if (verdict.state === 'incoherent') { prodLog.warn( `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + `The canonical tree or its counters diverged from the stamped state โ€” run ` + @@ -12564,6 +12589,92 @@ export class Brainy implements BrainyInterface { } } + /** + * @description THE TERMINAL VERDICT for a torn generation-log tail. + * + * A stamp whose `sourceGeneration` sits ABOVE the store's committed + * watermark witnesses a generation that is not in the log: the stamp's fsync + * outlived the tail's. By the time this runs, log-authority recovery has + * already folded every intact fact above the manifest and advanced the + * watermark to cover them โ€” so if the stamp is STILL ahead, the generation + * it names is not merely late, it is GONE. There is nothing to wait for. + * + * That is the whole point of this method. A field report of this class + * (single-process store, abrupt termination mid-fold) described a reopen + * that narrated the tear and then held 100% CPU with zero log growth for + * eight minutes before an operator wiped the directory. A recovery that + * cannot say what it is waiting for has no business spinning; the honest + * answer here is a verdict, taken now, at O(1) cost. + * + * WHAT THE VERDICT DOES โ€” the stamped surface is UNUSABLE, so it is + * discarded rather than believed: the stamped counts describe a generation + * that never became durable, and comparing them against live counters can + * only produce noise. The tree itself is not in question (it IS canonical โ€” + * every commit writes it, and the fold re-applied every after-image the log + * still holds), so the demotion is a re-derivation of this family's verified + * surface at the generation the store can actually show: + * + * - WRITER open โ†’ re-stamp at `committedGeneration()` from the live + * counters โ€” exactly what the next flush would write, taken now so the + * tear cannot re-narrate on every subsequent open. Both count sets are + * logged so an operator can see whether anything really moved. + * - READER open โ†’ a reader cannot re-stamp. Narrate the same terminal + * verdict with the named cure and carry on serving; a read-only inspector + * is never locked out of a store, and never left waiting either. + * + * BOUNDEDNESS: straight-line code. No loop, no retry, no await on any + * external progress signal โ€” the two counter reads and one stamp write are + * the entire cost, and none of them scales with the store. + */ + private async demoteTornEntityTreeStamp( + stamp: FamilyStamp, + stampSource: number, + head: number, + observed: { nounCount: number; verbCount: number } + ): Promise { + const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {} + const detail = + `[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` + + `generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` + + `generation is ${head} after crash recovery โ€” the stamp's fsync outlived the log tail's, ` + + `and generation ${stampSource} is not in the log to arrive. Stamped rollups ` + + `${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.` + + if (this.isReadOnly) { + prodLog.warn( + `${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` + + `family stays UNVERIFIED for this session (reads are unaffected โ€” the canonical tree ` + + `is the truth this stamp only describes). Cure: open the store with a writer, or run ` + + `brain.repairIndex() there, to recount from canonical and re-stamp.` + ) + return + } + + const startedAt = Date.now() + try { + await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { + family: 'entity-tree', + sourceGeneration: head, + members: { + mode: 'rollup', + invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount } + } + }) + prodLog.warn( + `${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` + + `from the live counters in ${Date.now() - startedAt}ms โ€” terminal, not a wait. If the ` + + `observed counts above look wrong for your data, run brain.repairIndex() to recount ` + + `from canonical.` + ) + } catch (error) { + prodLog.warn( + `${detail} The demotion's re-stamp FAILED (${(error as Error).message}) โ€” the tear will ` + + `narrate again at the next open, which is the honest outcome; the store still serves ` + + `from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.` + ) + } + } + /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts index 98342884..2f01e935 100644 --- a/src/db/familyStamp.ts +++ b/src/db/familyStamp.ts @@ -12,9 +12,11 @@ * the verified surface is a small set of rollup invariants (entity/ * relationship counts) plus `sourceGeneration`. * - * `sourceGeneration` is the generation of the source-of-truth log this - * projection reflects โ€” open-time coherence becomes a COMPARISON (stamp vs - * log head), not a walk: + * `sourceGeneration` is the COMMITTED generation of the source-of-truth log + * this projection reflects โ€” never the allocated counter, which names a + * generation that may never commit (see {@link StampVerdict.torn}) โ€” so + * open-time coherence becomes a COMPARISON (stamp vs committed head), not a + * walk: * * - equal + invariants hold โ†’ coherent, serve. * - behind โ†’ the projection missed the tail (crash between commit and stamp); @@ -24,6 +26,9 @@ * - invariants FAIL at equal generation โ†’ genuine incoherence: loud, and the * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * canonical walk) heals it. + * - AHEAD โ†’ a torn generation-log tail: the stamp's fsync outlived the log + * tail's. TERMINAL, never a wait โ€” the generation the stamp names does not + * exist to arrive. * * Stamps are JSON on purpose โ€” every incident gets debugged by reading a * stamp in a terminal. @@ -70,6 +75,12 @@ export type StampVerdict = | { state: 'coherent' } | { state: 'absent' } // legacy store โ€” first stamp writes at the next flush | { state: 'behind'; stampSource: number; head: number } + /** + * TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the + * store's committed watermark can no longer show. TERMINAL โ€” there is no + * generation to wait for, so the open demotes (or refuses) and never spins. + */ + | { state: 'torn'; stampSource: number; head: number } | { state: 'incoherent'; failures: string[] } | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp โ€” never conflated with absence @@ -118,12 +129,15 @@ export function verifyFamilyStamp( ): StampVerdict { if (stamp === null) return { state: 'absent' } if (stamp.sourceGeneration > head) { - // A stamp AHEAD of the log claims state that never committed โ€” the - // projection was stamped against truth that a crash rolled back. - return { - state: 'incoherent', - failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] - } + // A stamp AHEAD of committed truth witnesses a generation the store can no + // longer show: the stamp's fsync survived a crash that the log tail did + // not. This is the TORN GENERATION-LOG TAIL โ€” its own class, never folded + // in with `incoherent` (a count that drifted at a generation both sides + // agree on), because the two have opposite cures: incoherence is recounted, + // a tear is DEMOTED. It is also terminal by construction โ€” there is no + // generation the open can wait for, because the one the stamp names is + // gone. + return { state: 'torn', stampSource: stamp.sourceGeneration, head } } if (stamp.sourceGeneration < head) { return { state: 'behind', stampSource: stamp.sourceGeneration, head } diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts index deefc5e6..23cc0a15 100644 --- a/tests/integration/entity-tree-stamp.test.ts +++ b/tests/integration/entity-tree-stamp.test.ts @@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => { const invariants = (stamp.members as any).invariants expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) - expect(stamp.sourceGeneration).toBe(brain.generation()) + // THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the + // counter labelled the stamp with a generation a write in flight had merely + // claimed, so every crash inside a write window produced a spurious verdict + // at the next open (see the torn-tail pins below). + expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration()) expect(stamp.generation).toBeGreaterThanOrEqual(1) }) @@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => { expect(stillIncoherent).toEqual([]) }) + /** + * Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's + * committed watermark โ€” the durable shape a torn generation-log tail leaves + * behind (the stamp's fsync outlived the tail's). Fabricated rather than + * crash-produced so the pin is deterministic; the seeded-SIGKILL lane + * (`scripts/crash-consistency.mjs` in the engine repo) produces the same + * shape from a real abrupt termination. + */ + const fabricateTear = (ahead: number): FamilyStamp => { + const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`) + const zlib = require('node:zlib') + const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp + const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead } + fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn))) + return torn + } + + it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => { + for (let i = 0; i < 3; i++) + await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } }) + await brain.close() + const torn = fabricateTear(5) + + const warn = vi.spyOn(prodLog, 'warn') + const startedAt = Date.now() + brain = await open() + const openMs = Date.now() - startedAt + + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + // Narrated PRECISELY: both generations, the file, and the named cure. + expect(said).toContain(`source generation ${torn.sourceGeneration}`) + expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`) + expect(said).toContain(ENTITY_TREE_STAMP_PATH) + expect(said).toContain('DEMOTED') + expect(said).toMatch(/repairIndex\(\)/) + // Terminal, not a wait: the demotion is O(1) straight-line work, so a tear + // cannot turn an open into the 8-minute spin this class was reported as. + expect(openMs).toBeLessThan(30_000) + + // The store SERVES โ€” a tear in a stamp never locks an owner out of the + // canonical tree the stamp merely describes. + expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3) + + // The demotion CONVERGED: the stamp now names committed truth, and the + // next open is quiet. A verdict that re-narrates every open is a wait + // wearing a different hat. + const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp + expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration()) + await brain.close() + const warn2 = vi.spyOn(prodLog, 'warn') + brain = await open() + expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([]) + }) + + it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => { + await brain.add({ data: 'ro', type: 'document', metadata: {} }) + await brain.close() + const torn = fabricateTear(3) + + const warn = vi.spyOn(prodLog, 'warn') + const reader: any = await Brainy.openReadOnly({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + expect(said).toContain('READ-ONLY') + expect(said).toContain('UNVERIFIED') + expect(said).toMatch(/repairIndex\(\)/) + await reader.close() + + // A reader never rewrites the store: read the bytes back off disk (not + // through a writer open, which would demote them) โ€” the torn stamp is + // exactly as it was found. + const onDisk = JSON.parse( + require('node:zlib') + .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) + .toString('utf-8') + ) as FamilyStamp + expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration) + expect(onDisk.generation).toBe(torn.generation) + + brain = await open() + }) + it('the one verifier handles both member modes', () => { const rollup: FamilyStamp = { family: 'x', @@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => { stampSource: 5, head: 9 }) - expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head + // AHEAD is its own class โ€” a torn generation-log tail, never folded in + // with `incoherent`: the two have opposite cures (recount vs demote). + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({ + state: 'torn', + stampSource: 5, + head: 3 + }) expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) const enumerated: FamilyStamp = { From 9a888c37e9ebec5573cd7ebd0764396f3a424de3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:13:42 -0700 Subject: [PATCH 03/24] fix(generations): a sealed segment may only declare the generations it holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis of the "packed history is damaged" narration that fires on every run of the affected stores. It is a WRITER defect, and the reader's refusal was the symptom rather than the cause. A sealed segment declares one contiguous range [firstGeneration, lastGeneration], and every reader treats that range as containment: coveringSegment is an interval test, hasGeneration returns true for anything inside it, and open() seeds committedRanges from it. repackHistory handed fold() a SPARSE batch. Three filters punch holes in its candidate list mid-run โ€” a generation absent from committedRanges never appears, one still in the pending buffer is skipped, one whose tx.json will not read is skipped โ€” and fold() then computed the range from the first and last survivor, claiming every generation in between. The next open merged that mis-declared range back into committedRanges, re-admitting the hole as committed history, so the following auto-compaction pass asked the packed tier for a frame that was never written and failed. Re-merged at every open, which is why it repeated on every run. Confirmed against a forensic fixture: generation directories 1..2503 present except exactly one, 1416; and its fact-log segment already showed the tell โ€” seg-...1410.bfl declaring 1410..1940 (531 generations) while recording 530 facts. Three changes: - repackHistory folds each contiguous RUN as its own segment (`contiguousRuns`), so ranges describe exactly what the segments contain. - fold() REFUSES a non-contiguous batch, naming the gap and its width. The density law is now mechanical, so no future caller can reintroduce it. A refusal loses nothing: the generations stay live and readable. - Stores already carrying the damage heal instead of wedging. A segment whose declared span exceeds its frame count is SPARSE; `actualRanges()` reads the real generation list from its sidecar so open() never re-admits the holes, and readFrame reports such a hole as unpacked with a narration naming the segment, rather than throwing. A DENSE segment missing a frame is still loud damage โ€” that one means the manifest and sidecar disagree. Pins: nine unit cases (refusal and its message, honest ranges for separately folded runs, a reconstructed pre-fix sparse segment serving its real frames while reporting holes as unpacked, holes excluded from actualRanges, and the dense-segment damage path still throwing) plus an end-to-end case that deletes a generation directory and drives the real sequence โ€” ordinary close()-time repacking folds over the hole, then reopen and compact must both complete. Verified red without the fix: the segment declared an 11-generation span while holding 10 frames. --- src/db/generationSegments.ts | 119 +++++++++++++++++++- src/db/generationStore.ts | 66 +++++++++-- tests/integration/history-repacking.test.ts | 102 +++++++++++++++++ tests/unit/db/generation-segments.test.ts | 115 +++++++++++++++++++ 4 files changed, 389 insertions(+), 13 deletions(-) diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts index 0c14b60c..91451281 100644 --- a/src/db/generationSegments.ts +++ b/src/db/generationSegments.ts @@ -147,6 +147,60 @@ export class GenerationSegmentStore { return this.coveringSegment(gen) !== null } + /** + * @description True when `meta` declares more generations than it holds + * frames โ€” a segment sealed by a writer that folded across a hole. The + * manifest records `frames` at fold time, so this is an O(1) comparison + * against the declared span and needs no I/O. + */ + private isSparse(meta: SegmentMeta): boolean { + return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames + } + + /** + * @description The generations this tier ACTUALLY holds, as coalesced + * ascending intervals โ€” not what the segments declare. + * + * Dense segments (every one a current writer produces) contribute their + * declared range with no I/O. A SPARSE segment โ€” one sealed before the + * density law was enforced, whose declared range spans generations it has + * no frame for โ€” has its real generation list read from its sidecar and + * contributed instead, with the discrepancy narrated once. + * + * This is what keeps a store that already carries the damage from wedging. + * `open()` seeds `committedRanges` from these intervals, so a hole is never + * re-admitted as a committed generation, and the auto-compaction pass that + * used to fail on every run with "packed history is damaged" simply never + * asks for the missing frame. + * + * @returns Ascending, non-overlapping `[first, last]` intervals. + */ + async actualRanges(): Promise> { + const out: Array<[number, number]> = [] + for (const meta of this.manifest.segments) { + if (!this.isSparse(meta)) { + out.push([meta.firstGeneration, meta.lastGeneration]) + continue + } + const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames + prodLog.warn( + `[GenerationSegments] sealed segment ${meta.file} declares generations ` + + `${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` + + `frame(s) โ€” ${missing} generation(s) in that span were never folded into it. ` + + `Serving the frames it actually holds; the declared span is not treated as ` + + `committed history. (Written by a pre-density-law writer that folded across a ` + + `gap; the segment itself is intact and no record is lost.)` + ) + const idx = await this.sidecarFor(meta) + for (const [gen] of idx.generations) { + const last = out[out.length - 1] + if (last !== undefined && gen === last[1] + 1) last[1] = gen + else out.push([gen, gen]) + } + } + return out + } + /** * Fold consecutive generations into ONE new sealed segment + sidecar and * append it to the manifest atomically. Caller guarantees: `gens` is @@ -164,6 +218,38 @@ export class GenerationSegmentStore { throw new Error('[GenerationSegments] fold() input must be strictly ascending') } } + // THE DENSITY LAW, MADE MECHANICAL. + // + // A sealed segment declares a CONTIGUOUS range [firstGeneration, + // lastGeneration] and every reader treats that range as containment: + // `coveringSegment` is an interval test, `hasGeneration` returns true for + // anything inside it, and `open()` seeds committedRanges from it. So a + // segment folded from a SPARSE input silently claims generations it does + // not hold, and the first read of one of those holes throws + // "inside sealed segment ... but has no frame โ€” packed history is damaged". + // + // That is exactly how the damage was produced. `repackHistory` skipped + // generations mid-batch โ€” ones absent from committedRanges, ones still in + // the pending buffer, ones whose tx.json would not read โ€” and handed the + // survivors here, where the range was computed from the first and last of + // them. Worse, the mis-declared range was then merged back into + // committedRanges at the next open, which is what turned a quiet hole into + // a repeating auto-compaction failure on every subsequent run. + // + // Callers now split at discontinuities; this refusal is what keeps any + // future caller from reintroducing the class. A refusal here loses + // nothing โ€” the generations stay in the live tier, readable, and the next + // pass folds them correctly. + for (let i = 1; i < gens.length; i++) { + if (gens[i].generation !== gens[i - 1].generation + 1) { + throw new Error( + `[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} โ†’ ` + + `${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` + + `generation(s). A sealed segment declares a dense range, so folding a sparse ` + + `batch would claim generations it does not hold. Split the batch at the gap.` + ) + } + } const last = this.manifest.segments[this.manifest.segments.length - 1] if (last && gens[0].generation <= last.lastGeneration) { throw new Error( @@ -364,12 +450,37 @@ export class GenerationSegmentStore { return this.decodeFrame(payload) } } - // In the covering range but not present: the packed tier is dense by - // construction (fold packs every generation it is handed, including - // record-less ones) โ€” absence inside a sealed range is damage. + // Inside the covering range but with no frame. Two very different causes, + // and conflating them is what made this class wedge every maintenance pass + // on the affected stores. + // + // (1) A SPARSE SEGMENT โ€” the manifest's own `frames` count is smaller than + // the span it declares. That segment was sealed by a writer that + // folded across a hole (the class this file's density law now bars). + // The segment is INTACT and nothing is lost; it simply never held this + // generation. Answering "not packed" is the honest answer, and it lets + // the caller's two-tier read decide what a genuinely absent generation + // means, instead of every compaction pass dying on a repeating throw. + // `actualRanges()` keeps such holes out of committedRanges at open, so + // in a healed store nobody asks this question in the first place. + // + // (2) A DENSE SEGMENT missing a frame it says it has โ€” the manifest and + // the sidecar disagree about a segment that claims to be complete. + // That IS damage, and it stays loud. + if (this.isSparse(meta)) { + prodLog.warn( + `[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` + + `declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` + + `holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` + + `-generation span โ€” it was sealed across a gap and never held this generation. ` + + `Reporting it as unpacked rather than as damage; no record is lost.` + ) + return null + } throw new Error( `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame โ€” packed history is damaged` + `range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + + `span โ€” the manifest and the sidecar disagree; packed history is damaged` ) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index fd052c31..da21dc61 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -96,6 +96,35 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' +/** + * @description Split an ascending list of fold candidates into maximal + * CONTIGUOUS runs โ€” `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`. + * + * A sealed segment declares one dense range `[firstGeneration, + * lastGeneration]`, and every reader treats that range as containment. So a + * batch with a hole in it must never become one segment: it would claim a + * generation it does not hold, and the first read of that hole reports the + * packed history as damaged. One run, one segment โ€” the ranges then describe + * exactly what the segments contain. + * + * @param gens - Fold candidates, strictly ascending by generation. + * @returns One array per contiguous run, in ascending order. Empty in, empty out. + */ +export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] { + const runs: FoldGeneration[][] = [] + let run: FoldGeneration[] = [] + for (const g of gens) { + const prev = run[run.length - 1] + if (prev !== undefined && g.generation !== prev.generation + 1) { + runs.push(run) + run = [] + } + run.push(g) + } + if (run.length > 0) runs.push(run) + return runs +} + /** * @description Phases of the {@link GenerationStore.commitTransaction} commit * protocol at which a test-only fault injector can simulate a process crash. @@ -784,9 +813,15 @@ export class GenerationStore { if (storageSupportsFactLog(this.storage)) { this.segments = new GenerationSegmentStore(this.storage) await this.segments.open() - const packedRanges = this.segments - .segments() - .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) + // ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law + // writer can declare a span wider than the frames it holds; seeding + // committedRanges from the declared span re-admits those holes as + // committed generations, and every later maintenance pass then asks for a + // frame that was never written. `actualRanges()` reads the real + // generation list from the sidecar for exactly those segments (and does + // no I/O for the dense ones, which is all of them on a healthy store). + const packedRanges = (await this.segments.actualRanges()) + .map((r): [number, number] => [r[0], Math.min(r[1], this.committed)]) .filter(([lo, hi]) => lo <= hi) if (packedRanges.length > 0) { // Merge packed (older) + live (newer) interval sets โ€” both ascending; @@ -3121,13 +3156,26 @@ export class GenerationStore { foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) } if (foldInput.length === 0) continue - await segments.fold(foldInput) - segmentsCreated++ - // Segment + manifest durable โ†’ the live copies retire. - for (const g of foldInput) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + // SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous โ€” three + // filters above punch holes in it: a generation missing from + // committedRanges never appears, one still in the pending buffer is + // skipped, and one whose tx.json will not read is skipped. A sealed + // segment declares a DENSE range, so folding across such a hole makes + // the segment claim a generation it does not hold; the next open + // merges that mis-declared range into committedRanges, and every + // subsequent auto-compaction pass then asks for the missing frame and + // fails with "packed history is damaged". Fold each contiguous RUN as + // its own segment instead โ€” same bytes, honest ranges. + for (const run of contiguousRuns(foldInput)) { + if (deadline !== undefined && Date.now() >= deadline) break + await segments.fold(run) + segmentsCreated++ + // Segment + manifest durable โ†’ the live copies retire. + for (const g of run) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + } + folded += run.length } - folded += foldInput.length } if (folded > 0) { prodLog.info( diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts index 2bcee038..bb07268d 100644 --- a/tests/integration/history-repacking.test.ts +++ b/tests/integration/history-repacking.test.ts @@ -16,6 +16,7 @@ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as path from 'node:path' import * as os from 'node:os' +import * as zlib from 'node:zlib' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { GenerationStore } from '../../src/db/generationStore.js' @@ -57,6 +58,107 @@ describe('history repacking โ€” the two-tier lifecycle', () => { } }) + /** + * THE HOLE, END TO END โ€” the shape a real store carries. + * + * A forensic fixture was measured with generation directories 1..2503 + * present except for exactly one: 1416. Its fact-log segment already showed + * the tell โ€” `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration + * 1940 (531 generations) while recording only 530 facts. + * + * Before the fix, repacking such a store folded ACROSS that hole: the batch + * skipped 1416 (no readable delta) and the sealed segment declared a range + * spanning it anyway. The next open merged that declared range back into + * committedRanges, re-admitting 1416 as committed history, and every + * subsequent auto-compaction pass then asked the packed tier for a frame + * that was never written โ€” producing, on EVERY run, the non-fatal narration + * + * Auto-compaction of generational history failed (non-fatal): generation + * N is inside sealed segment seg-....bgs's declared range but has no frame + * โ€” packed history is damaged + * + * This pin removes a generation directory to make the same hole, then + * requires repack + reopen + compaction to complete cleanly. + */ + it('a missing generation directory does not poison the packed tier', async () => { + const dir = tempDir() + // `retention: 'all'` throughout: close() otherwise auto-compacts the + // history away, and this pin needs the cold generations still on disk so + // there is something to punch a hole in. The live window stays at its + // production default for the build phase, so nothing folds yet. + const archival = async (): Promise => { + const b = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + embeddingFunction: stub, + retention: 'all' + }) + await b.init() + return b + } + const brain = await archival() + + const id = await brain.add({ + data: 'holed-entity', + type: NounType.Document, + metadata: { v: 0 } + }) + // One flush per update: single-op writes coalesce inside a flush window, + // so a history deep enough to have a middle needs the windows separated. + for (let v = 1; v <= 12; v++) { + await brain.update({ id, metadata: { v } }) + await brain.flush() + } + await brain.close() + + // Punch the hole: delete ONE generation directory in the middle of the + // cold range, exactly as the real store presents it. + const genRoot = path.join(dir, '_generations') + const numeric = fs + .readdirSync(genRoot, { withFileTypes: true }) + .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)) + .map((e) => Number(e.name)) + .sort((a, b) => a - b) + expect(numeric.length).toBeGreaterThan(6) + const victim = numeric[Math.floor(numeric.length / 2)] + fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true }) + + // Now shrink the live window and reopen. close() repacks automatically + // (brainy.ts phase 0b), so this is the production sequence exactly: a + // store with a hole in its history gets folded by ordinary housekeeping, + // with nobody asking for it. + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 + const reopened = await archival() + const result = await reopened.repackHistory() + expect(result.foldedGenerations).toBeGreaterThan(0) + + const segDir = path.join(dir, SEGMENTS_PREFIX) + const manifestPath = ['manifest.json', 'manifest.json.gz'] + .map((f) => path.join(segDir, f)) + .find((p) => fs.existsSync(p))! + const raw = manifestPath.endsWith('.gz') + ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8') + : fs.readFileSync(manifestPath, 'utf8') + const manifest = JSON.parse(raw) as { + segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }> + } + + // THE LAW: every sealed segment declares exactly as many generations as it + // holds frames, and none of them spans the victim. + for (const s of manifest.segments) { + expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames) + expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false) + } + + await reopened.close() + + // And the pass that used to fail on every run now completes: reopen (which + // re-seeds committedRanges from the packed tier) then compact history. + const third = await openBrain(dir) + await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined() + await third.close() + }) + it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 const dir = tempDir() diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts index 27ab85cb..f16e67b3 100644 --- a/tests/unit/db/generation-segments.test.ts +++ b/tests/unit/db/generation-segments.test.ts @@ -147,4 +147,119 @@ describe('db/GenerationSegmentStore โ€” the D1+D3 packed tier', () => { await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) + + // ========================================================================== + // THE DENSITY LAW + // ========================================================================== + // + // A sealed segment declares a CONTIGUOUS range and every reader treats that + // range as containment. Folding a sparse batch therefore makes the segment + // claim generations it does not hold โ€” and because `open()` merges declared + // ranges back into committedRanges, the hole is re-admitted as committed + // history and every later maintenance pass fails asking for a frame that was + // never written. That is the "generation N is inside sealed segment + // seg-....bgs's declared range but has no frame โ€” packed history is damaged" + // narration seen on every run of the affected stores. + + it('fold REFUSES a batch with a hole โ€” a dense range may not be declared over sparse input', async () => { + await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow( + /not contiguous: 2 โ†’ 4 skips 1 generation/ + ) + // The refusal loses nothing: no segment was sealed, so the generations + // stay in the live tier and the next pass folds them correctly. + expect(store.segments()).toHaveLength(0) + expect(store.hasGeneration(1)).toBe(false) + }) + + it('a wider gap names how many generations it would have swallowed', async () => { + await expect(store.fold([gen(10), gen(20)])).rejects.toThrow( + /not contiguous: 10 โ†’ 20 skips 9 generation\(s\)/ + ) + }) + + it('two contiguous runs folded separately declare honest ranges', async () => { + // What the caller now does instead of folding across the gap. + const a = await store.fold([gen(1), gen(2), gen(3)]) + const b = await store.fold([gen(7), gen(8)]) + expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) + expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 }) + // The gap is honestly outside the packed tier. + for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false) + for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true) + expect(await store.actualRanges()).toEqual([ + [1, 3], + [7, 8] + ]) + }) + + it('actualRanges() is exact and I/O-free for dense segments', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + // Adjacent dense segments each contribute their declared range. + expect(await store.actualRanges()).toEqual([ + [1, 2], + [3, 4] + ]) + }) + + // ---- pre-existing damage: a store sealed by the old writer ---------------- + + /** + * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and + * sidecar for a contiguous run, then rewrite the manifest so the segment + * declares a wider range than the frames it holds. This reproduces on disk + * exactly what the affected stores carry, without needing the old code. + */ + const sealSparseSegment = async (): Promise => { + await store.fold([gen(1), gen(2), gen(3)]) + const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any + // Declare 1..5 while holding frames for 1..3 โ€” generations 4 and 5 become + // holes inside a sealed range. + manifest.segments[0].lastGeneration = 5 + await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest) + } + + it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + + // The frames it really holds still serve, byte-faithfully. + expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002) + expect(await reopened.readRecords(3)).toHaveLength(2) + + // The holes answer "not packed" instead of throwing. This is the fix for + // the wedge: the old reader threw here on EVERY maintenance pass. + expect(await reopened.readDelta(4)).toBeNull() + expect(await reopened.readRecords(5)).toBeNull() + }) + + it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + // Declared 1..5; actually holds 1..3. The store seeds committedRanges from + // THIS, so generations 4 and 5 never become committed history again. + expect(await reopened.actualRanges()).toEqual([[1, 3]]) + }) + + it('a DENSE segment missing a frame is still loud damage', async () => { + // The other side of the branch: when the manifest claims a complete span, + // a missing frame means the manifest and sidecar disagree โ€” real damage, + // and it must not be quietly downgraded to "unpacked". + await store.fold([gen(1), gen(2), gen(3)]) + const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` + const raw = (await storage.readRawBytes(idxPath))! + const { decode, encode } = await import('@msgpack/msgpack') + const idx = decode(raw) as any + // Drop generation 2's entry while the manifest still declares 3 frames. + idx.generations = idx.generations.filter(([g]: [number]) => g !== 2) + await storage.writeRawBytes(idxPath, encode(idx)) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + await expect(reopened.readDelta(2)).rejects.toThrow( + /manifest and the sidecar disagree; packed history is damaged/ + ) + }) }) From 655aa13ea79e23927cde7fd47ab13b505cb042d9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:30:46 -0700 Subject: [PATCH 04/24] =?UTF-8?q?build(release):=20the=20docs-push=20step?= =?UTF-8?q?=20retires=20=E2=80=94=20this=20engine=20documents=20itself=20i?= =?UTF-8?q?n=20its=20own=20repository?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one-doc-set ruling (2026-08-31) gives soulcraft.com/docs to the paid product alone; the site serves redirects for the slugs this rail used to push. The push script stays in the tree as history; the rail stops calling it. --- scripts/release.sh | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 67ad1053..5d434320 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -248,17 +248,12 @@ else echo -e "${RED}โš ๏ธ FORGEJO_RELEASE_TOKEN unset โ€” no release page created; tag + CHANGELOG remain the record${NC}\n" fi -# Step 12: Push public docs to the soulcraft.com docs ingest door -# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when -# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish โ€” -# that already happened) when a push errors, so the docs site never -# silently trails npm. -echo -e "${BLUE}1๏ธโƒฃ2๏ธโƒฃ Pushing public docs to soulcraft.com/docs...${NC}" -if node scripts/push-docs.js; then - echo -e "${GREEN}โœ… Docs push step done${NC}\n" -else - echo -e "${RED}โŒ Docs push FAILED โ€” soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" -fi +# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): +# soulcraft.com/docs carries the paid product's documentation only. This +# engine's documentation home is THIS repository โ€” README and docs/ โ€” and the +# site serves 301s for the slugs this rail used to push. The push script stays +# in the tree for history; the rail no longer calls it. +echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" echo -e "${GREEN}โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”${NC}" echo -e "${GREEN}๐ŸŽ‰ Release ${NEW_VERSION} complete!${NC}" From c99308710aa5030d80bef794354216851443916c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:07:18 -0700 Subject: [PATCH 05/24] fix(recovery): a torn generation-log tail is a terminal verdict, never a wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of one defect, found by a seeded-SIGKILL crash lane. THE FALSE POSITIVE. stampEntityTree() recorded generationStore.generation() โ€” the ALLOCATED counter, a number a write in flight has claimed and may never commit โ€” while the JSDoc beside it already said the source is the committed generation. Every crash inside a write window therefore produced a spurious verdict at the next open: either 'sourceGeneration N is ahead of the log head N-1' (the allocated generation died with the process) or 'rollup invariant nounCount: stamped X, observed Y' (the recovery fold folded facts the stamp's counts predate). Both told the operator to run repairIndex() โ€” a whole-store recount โ€” for a store that was coherent. Measured before this commit: 4 of 11 SIGKILL cycles on a healthy store raised one of the two. The stamp and the open now both read committedGeneration(), which is what every other open-time watermark in the class already reasons about. THE TERMINAL VERDICT. A stamp still ahead of committed truth after the recovery fold witnesses a generation that is not in the log โ€” the stamp's fsync outlived the tail's, and there is nothing to arrive. That is its own verdict state now ('torn'), never folded in with 'incoherent': the two have opposite cures. A writer open demotes it โ€” the unusable stamped surface is re-derived at the committed generation from the live counters, O(1), straight-line, no loop and no await on external progress, narrated with both count sets, the stamp's path and its committedAt. A read-only open cannot re-stamp, so it says so and names the cure instead of guessing, and still serves. Neither branch waits, and neither locks an owner out of a canonical tree the stamp only describes. Pins: the verifier returns the torn verdict with both generations; a fabricated head-behind-source store narrates precisely, demotes inside a bounded open, serves its rows, and is quiet at the next open (the demotion converges); a read-only open narrates the same verdict and leaves the bytes untouched. (cherry picked from commit 298cb6dacaac9ef80db65a723d65cbd53c15d23e) --- src/brainy.ts | 121 +++++++++++++++++++- src/db/familyStamp.ts | 32 ++++-- tests/integration/entity-tree-stamp.test.ts | 104 ++++++++++++++++- 3 files changed, 241 insertions(+), 16 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 70d46973..da04577e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -12497,6 +12497,18 @@ export class Brainy implements BrainyInterface { * healed by `repairIndex()`, whose unconditional recount rebuilds the * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write * fault warns loudly but never fails the flush that carried real data. + * + * THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is + * the ALLOCATED counter โ€” a number a write in flight has claimed and may + * never commit. Stamping it made the stamp's generation label a claim about + * counts it was not taken at, and every crash inside a write window then + * produced a spurious verdict at the next open: either `sourceGeneration N + * is ahead of the log head N-1` (the allocated generation died with the + * process) or `rollup invariant 'nounCount': stamped X, observed Y` (the + * recovery fold folded facts the stamp's counts predate). MEASURED on the + * crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on + * a coherent store raised one of those two verdicts, each of them naming + * `repairIndex()` โ€” a whole-store recount โ€” as the cure for nothing. */ private async stampEntityTree(): Promise { if (this.isReadOnly) return @@ -12507,7 +12519,7 @@ export class Brainy implements BrainyInterface { ]) await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { family: 'entity-tree', - sourceGeneration: this.generationStore.generation(), + sourceGeneration: this.generationStore.committedGeneration(), members: { mode: 'rollup', invariants: { nounCount, verbCount } } }) } catch (error) { @@ -12520,16 +12532,24 @@ export class Brainy implements BrainyInterface { /** * @description Open-time coherence check for the entity tree's family stamp: - * compare `sourceGeneration` against the log head and the stamped rollup - * invariants against the live counters. Verdicts: + * compare `sourceGeneration` against the store's COMMITTED generation and + * the stamped rollup invariants against the live counters. Verdicts: * - `coherent` / `absent` (legacy store; first flush stamps) โ†’ silent. * - `behind` โ†’ benign for the tree (it is written BY the commit; only the * stamp is stale โ€” a crash landed between commit and flush). Refreshed at * the next flush. + * - `torn` โ†’ a TORN GENERATION-LOG TAIL, handled by + * {@link demoteTornEntityTreeStamp}: terminal, never a wait. * - `incoherent` โ†’ LOUD: the tree or its counters diverged from what was * stamped โ€” `repairIndex()` recounts from canonical and re-stamps. * Never blocks open; a fault reading the stamp is surfaced as unverifiable, * never conflated with absence. + * + * THE COMPARISON IS AGAINST `committedGeneration()`, matching what + * {@link stampEntityTree} writes and what every other open-time watermark in + * this class already reasons about (the fact-scan capability, the metadata / + * graph / HNSW watermark verdicts). Comparing against the allocated counter + * was the one place that disagreed, and disagreeing was the whole defect. */ private async verifyEntityTreeStamp(): Promise { let stamp: FamilyStamp | null @@ -12546,11 +12566,16 @@ export class Brainy implements BrainyInterface { this.storage.getNounCount(), this.storage.getVerbCount() ]) - const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { + const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), { nounCount, verbCount }) - if (verdict.state === 'incoherent') { + if (verdict.state === 'torn') { + await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, { + nounCount, + verbCount + }) + } else if (verdict.state === 'incoherent') { prodLog.warn( `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + `The canonical tree or its counters diverged from the stamped state โ€” run ` + @@ -12564,6 +12589,92 @@ export class Brainy implements BrainyInterface { } } + /** + * @description THE TERMINAL VERDICT for a torn generation-log tail. + * + * A stamp whose `sourceGeneration` sits ABOVE the store's committed + * watermark witnesses a generation that is not in the log: the stamp's fsync + * outlived the tail's. By the time this runs, log-authority recovery has + * already folded every intact fact above the manifest and advanced the + * watermark to cover them โ€” so if the stamp is STILL ahead, the generation + * it names is not merely late, it is GONE. There is nothing to wait for. + * + * That is the whole point of this method. A field report of this class + * (single-process store, abrupt termination mid-fold) described a reopen + * that narrated the tear and then held 100% CPU with zero log growth for + * eight minutes before an operator wiped the directory. A recovery that + * cannot say what it is waiting for has no business spinning; the honest + * answer here is a verdict, taken now, at O(1) cost. + * + * WHAT THE VERDICT DOES โ€” the stamped surface is UNUSABLE, so it is + * discarded rather than believed: the stamped counts describe a generation + * that never became durable, and comparing them against live counters can + * only produce noise. The tree itself is not in question (it IS canonical โ€” + * every commit writes it, and the fold re-applied every after-image the log + * still holds), so the demotion is a re-derivation of this family's verified + * surface at the generation the store can actually show: + * + * - WRITER open โ†’ re-stamp at `committedGeneration()` from the live + * counters โ€” exactly what the next flush would write, taken now so the + * tear cannot re-narrate on every subsequent open. Both count sets are + * logged so an operator can see whether anything really moved. + * - READER open โ†’ a reader cannot re-stamp. Narrate the same terminal + * verdict with the named cure and carry on serving; a read-only inspector + * is never locked out of a store, and never left waiting either. + * + * BOUNDEDNESS: straight-line code. No loop, no retry, no await on any + * external progress signal โ€” the two counter reads and one stamp write are + * the entire cost, and none of them scales with the store. + */ + private async demoteTornEntityTreeStamp( + stamp: FamilyStamp, + stampSource: number, + head: number, + observed: { nounCount: number; verbCount: number } + ): Promise { + const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {} + const detail = + `[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` + + `generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` + + `generation is ${head} after crash recovery โ€” the stamp's fsync outlived the log tail's, ` + + `and generation ${stampSource} is not in the log to arrive. Stamped rollups ` + + `${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.` + + if (this.isReadOnly) { + prodLog.warn( + `${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` + + `family stays UNVERIFIED for this session (reads are unaffected โ€” the canonical tree ` + + `is the truth this stamp only describes). Cure: open the store with a writer, or run ` + + `brain.repairIndex() there, to recount from canonical and re-stamp.` + ) + return + } + + const startedAt = Date.now() + try { + await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { + family: 'entity-tree', + sourceGeneration: head, + members: { + mode: 'rollup', + invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount } + } + }) + prodLog.warn( + `${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` + + `from the live counters in ${Date.now() - startedAt}ms โ€” terminal, not a wait. If the ` + + `observed counts above look wrong for your data, run brain.repairIndex() to recount ` + + `from canonical.` + ) + } catch (error) { + prodLog.warn( + `${detail} The demotion's re-stamp FAILED (${(error as Error).message}) โ€” the tear will ` + + `narrate again at the next open, which is the honest outcome; the store still serves ` + + `from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.` + ) + } + } + /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts index 98342884..2f01e935 100644 --- a/src/db/familyStamp.ts +++ b/src/db/familyStamp.ts @@ -12,9 +12,11 @@ * the verified surface is a small set of rollup invariants (entity/ * relationship counts) plus `sourceGeneration`. * - * `sourceGeneration` is the generation of the source-of-truth log this - * projection reflects โ€” open-time coherence becomes a COMPARISON (stamp vs - * log head), not a walk: + * `sourceGeneration` is the COMMITTED generation of the source-of-truth log + * this projection reflects โ€” never the allocated counter, which names a + * generation that may never commit (see {@link StampVerdict.torn}) โ€” so + * open-time coherence becomes a COMPARISON (stamp vs committed head), not a + * walk: * * - equal + invariants hold โ†’ coherent, serve. * - behind โ†’ the projection missed the tail (crash between commit and stamp); @@ -24,6 +26,9 @@ * - invariants FAIL at equal generation โ†’ genuine incoherence: loud, and the * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * canonical walk) heals it. + * - AHEAD โ†’ a torn generation-log tail: the stamp's fsync outlived the log + * tail's. TERMINAL, never a wait โ€” the generation the stamp names does not + * exist to arrive. * * Stamps are JSON on purpose โ€” every incident gets debugged by reading a * stamp in a terminal. @@ -70,6 +75,12 @@ export type StampVerdict = | { state: 'coherent' } | { state: 'absent' } // legacy store โ€” first stamp writes at the next flush | { state: 'behind'; stampSource: number; head: number } + /** + * TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the + * store's committed watermark can no longer show. TERMINAL โ€” there is no + * generation to wait for, so the open demotes (or refuses) and never spins. + */ + | { state: 'torn'; stampSource: number; head: number } | { state: 'incoherent'; failures: string[] } | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp โ€” never conflated with absence @@ -118,12 +129,15 @@ export function verifyFamilyStamp( ): StampVerdict { if (stamp === null) return { state: 'absent' } if (stamp.sourceGeneration > head) { - // A stamp AHEAD of the log claims state that never committed โ€” the - // projection was stamped against truth that a crash rolled back. - return { - state: 'incoherent', - failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] - } + // A stamp AHEAD of committed truth witnesses a generation the store can no + // longer show: the stamp's fsync survived a crash that the log tail did + // not. This is the TORN GENERATION-LOG TAIL โ€” its own class, never folded + // in with `incoherent` (a count that drifted at a generation both sides + // agree on), because the two have opposite cures: incoherence is recounted, + // a tear is DEMOTED. It is also terminal by construction โ€” there is no + // generation the open can wait for, because the one the stamp names is + // gone. + return { state: 'torn', stampSource: stamp.sourceGeneration, head } } if (stamp.sourceGeneration < head) { return { state: 'behind', stampSource: stamp.sourceGeneration, head } diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts index deefc5e6..23cc0a15 100644 --- a/tests/integration/entity-tree-stamp.test.ts +++ b/tests/integration/entity-tree-stamp.test.ts @@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => { const invariants = (stamp.members as any).invariants expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) - expect(stamp.sourceGeneration).toBe(brain.generation()) + // THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the + // counter labelled the stamp with a generation a write in flight had merely + // claimed, so every crash inside a write window produced a spurious verdict + // at the next open (see the torn-tail pins below). + expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration()) expect(stamp.generation).toBeGreaterThanOrEqual(1) }) @@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => { expect(stillIncoherent).toEqual([]) }) + /** + * Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's + * committed watermark โ€” the durable shape a torn generation-log tail leaves + * behind (the stamp's fsync outlived the tail's). Fabricated rather than + * crash-produced so the pin is deterministic; the seeded-SIGKILL lane + * (`scripts/crash-consistency.mjs` in the engine repo) produces the same + * shape from a real abrupt termination. + */ + const fabricateTear = (ahead: number): FamilyStamp => { + const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`) + const zlib = require('node:zlib') + const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp + const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead } + fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn))) + return torn + } + + it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => { + for (let i = 0; i < 3; i++) + await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } }) + await brain.close() + const torn = fabricateTear(5) + + const warn = vi.spyOn(prodLog, 'warn') + const startedAt = Date.now() + brain = await open() + const openMs = Date.now() - startedAt + + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + // Narrated PRECISELY: both generations, the file, and the named cure. + expect(said).toContain(`source generation ${torn.sourceGeneration}`) + expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`) + expect(said).toContain(ENTITY_TREE_STAMP_PATH) + expect(said).toContain('DEMOTED') + expect(said).toMatch(/repairIndex\(\)/) + // Terminal, not a wait: the demotion is O(1) straight-line work, so a tear + // cannot turn an open into the 8-minute spin this class was reported as. + expect(openMs).toBeLessThan(30_000) + + // The store SERVES โ€” a tear in a stamp never locks an owner out of the + // canonical tree the stamp merely describes. + expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3) + + // The demotion CONVERGED: the stamp now names committed truth, and the + // next open is quiet. A verdict that re-narrates every open is a wait + // wearing a different hat. + const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp + expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration()) + await brain.close() + const warn2 = vi.spyOn(prodLog, 'warn') + brain = await open() + expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([]) + }) + + it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => { + await brain.add({ data: 'ro', type: 'document', metadata: {} }) + await brain.close() + const torn = fabricateTear(3) + + const warn = vi.spyOn(prodLog, 'warn') + const reader: any = await Brainy.openReadOnly({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + expect(said).toContain('READ-ONLY') + expect(said).toContain('UNVERIFIED') + expect(said).toMatch(/repairIndex\(\)/) + await reader.close() + + // A reader never rewrites the store: read the bytes back off disk (not + // through a writer open, which would demote them) โ€” the torn stamp is + // exactly as it was found. + const onDisk = JSON.parse( + require('node:zlib') + .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) + .toString('utf-8') + ) as FamilyStamp + expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration) + expect(onDisk.generation).toBe(torn.generation) + + brain = await open() + }) + it('the one verifier handles both member modes', () => { const rollup: FamilyStamp = { family: 'x', @@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => { stampSource: 5, head: 9 }) - expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head + // AHEAD is its own class โ€” a torn generation-log tail, never folded in + // with `incoherent`: the two have opposite cures (recount vs demote). + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({ + state: 'torn', + stampSource: 5, + head: 3 + }) expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) const enumerated: FamilyStamp = { From a963a744ccf668edc440c76d7f79fd0b216522c1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:13:42 -0700 Subject: [PATCH 06/24] fix(generations): a sealed segment may only declare the generations it holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis of the "packed history is damaged" narration that fires on every run of the affected stores. It is a WRITER defect, and the reader's refusal was the symptom rather than the cause. A sealed segment declares one contiguous range [firstGeneration, lastGeneration], and every reader treats that range as containment: coveringSegment is an interval test, hasGeneration returns true for anything inside it, and open() seeds committedRanges from it. repackHistory handed fold() a SPARSE batch. Three filters punch holes in its candidate list mid-run โ€” a generation absent from committedRanges never appears, one still in the pending buffer is skipped, one whose tx.json will not read is skipped โ€” and fold() then computed the range from the first and last survivor, claiming every generation in between. The next open merged that mis-declared range back into committedRanges, re-admitting the hole as committed history, so the following auto-compaction pass asked the packed tier for a frame that was never written and failed. Re-merged at every open, which is why it repeated on every run. Confirmed against a forensic fixture: generation directories 1..2503 present except exactly one, 1416; and its fact-log segment already showed the tell โ€” seg-...1410.bfl declaring 1410..1940 (531 generations) while recording 530 facts. Three changes: - repackHistory folds each contiguous RUN as its own segment (`contiguousRuns`), so ranges describe exactly what the segments contain. - fold() REFUSES a non-contiguous batch, naming the gap and its width. The density law is now mechanical, so no future caller can reintroduce it. A refusal loses nothing: the generations stay live and readable. - Stores already carrying the damage heal instead of wedging. A segment whose declared span exceeds its frame count is SPARSE; `actualRanges()` reads the real generation list from its sidecar so open() never re-admits the holes, and readFrame reports such a hole as unpacked with a narration naming the segment, rather than throwing. A DENSE segment missing a frame is still loud damage โ€” that one means the manifest and sidecar disagree. Pins: nine unit cases (refusal and its message, honest ranges for separately folded runs, a reconstructed pre-fix sparse segment serving its real frames while reporting holes as unpacked, holes excluded from actualRanges, and the dense-segment damage path still throwing) plus an end-to-end case that deletes a generation directory and drives the real sequence โ€” ordinary close()-time repacking folds over the hole, then reopen and compact must both complete. Verified red without the fix: the segment declared an 11-generation span while holding 10 frames. (cherry picked from commit 9a888c37e9ebec5573cd7ebd0764396f3a424de3) --- src/db/generationSegments.ts | 119 +++++++++++++++++++- src/db/generationStore.ts | 66 +++++++++-- tests/integration/history-repacking.test.ts | 102 +++++++++++++++++ tests/unit/db/generation-segments.test.ts | 115 +++++++++++++++++++ 4 files changed, 389 insertions(+), 13 deletions(-) diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts index 0c14b60c..91451281 100644 --- a/src/db/generationSegments.ts +++ b/src/db/generationSegments.ts @@ -147,6 +147,60 @@ export class GenerationSegmentStore { return this.coveringSegment(gen) !== null } + /** + * @description True when `meta` declares more generations than it holds + * frames โ€” a segment sealed by a writer that folded across a hole. The + * manifest records `frames` at fold time, so this is an O(1) comparison + * against the declared span and needs no I/O. + */ + private isSparse(meta: SegmentMeta): boolean { + return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames + } + + /** + * @description The generations this tier ACTUALLY holds, as coalesced + * ascending intervals โ€” not what the segments declare. + * + * Dense segments (every one a current writer produces) contribute their + * declared range with no I/O. A SPARSE segment โ€” one sealed before the + * density law was enforced, whose declared range spans generations it has + * no frame for โ€” has its real generation list read from its sidecar and + * contributed instead, with the discrepancy narrated once. + * + * This is what keeps a store that already carries the damage from wedging. + * `open()` seeds `committedRanges` from these intervals, so a hole is never + * re-admitted as a committed generation, and the auto-compaction pass that + * used to fail on every run with "packed history is damaged" simply never + * asks for the missing frame. + * + * @returns Ascending, non-overlapping `[first, last]` intervals. + */ + async actualRanges(): Promise> { + const out: Array<[number, number]> = [] + for (const meta of this.manifest.segments) { + if (!this.isSparse(meta)) { + out.push([meta.firstGeneration, meta.lastGeneration]) + continue + } + const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames + prodLog.warn( + `[GenerationSegments] sealed segment ${meta.file} declares generations ` + + `${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` + + `frame(s) โ€” ${missing} generation(s) in that span were never folded into it. ` + + `Serving the frames it actually holds; the declared span is not treated as ` + + `committed history. (Written by a pre-density-law writer that folded across a ` + + `gap; the segment itself is intact and no record is lost.)` + ) + const idx = await this.sidecarFor(meta) + for (const [gen] of idx.generations) { + const last = out[out.length - 1] + if (last !== undefined && gen === last[1] + 1) last[1] = gen + else out.push([gen, gen]) + } + } + return out + } + /** * Fold consecutive generations into ONE new sealed segment + sidecar and * append it to the manifest atomically. Caller guarantees: `gens` is @@ -164,6 +218,38 @@ export class GenerationSegmentStore { throw new Error('[GenerationSegments] fold() input must be strictly ascending') } } + // THE DENSITY LAW, MADE MECHANICAL. + // + // A sealed segment declares a CONTIGUOUS range [firstGeneration, + // lastGeneration] and every reader treats that range as containment: + // `coveringSegment` is an interval test, `hasGeneration` returns true for + // anything inside it, and `open()` seeds committedRanges from it. So a + // segment folded from a SPARSE input silently claims generations it does + // not hold, and the first read of one of those holes throws + // "inside sealed segment ... but has no frame โ€” packed history is damaged". + // + // That is exactly how the damage was produced. `repackHistory` skipped + // generations mid-batch โ€” ones absent from committedRanges, ones still in + // the pending buffer, ones whose tx.json would not read โ€” and handed the + // survivors here, where the range was computed from the first and last of + // them. Worse, the mis-declared range was then merged back into + // committedRanges at the next open, which is what turned a quiet hole into + // a repeating auto-compaction failure on every subsequent run. + // + // Callers now split at discontinuities; this refusal is what keeps any + // future caller from reintroducing the class. A refusal here loses + // nothing โ€” the generations stay in the live tier, readable, and the next + // pass folds them correctly. + for (let i = 1; i < gens.length; i++) { + if (gens[i].generation !== gens[i - 1].generation + 1) { + throw new Error( + `[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} โ†’ ` + + `${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` + + `generation(s). A sealed segment declares a dense range, so folding a sparse ` + + `batch would claim generations it does not hold. Split the batch at the gap.` + ) + } + } const last = this.manifest.segments[this.manifest.segments.length - 1] if (last && gens[0].generation <= last.lastGeneration) { throw new Error( @@ -364,12 +450,37 @@ export class GenerationSegmentStore { return this.decodeFrame(payload) } } - // In the covering range but not present: the packed tier is dense by - // construction (fold packs every generation it is handed, including - // record-less ones) โ€” absence inside a sealed range is damage. + // Inside the covering range but with no frame. Two very different causes, + // and conflating them is what made this class wedge every maintenance pass + // on the affected stores. + // + // (1) A SPARSE SEGMENT โ€” the manifest's own `frames` count is smaller than + // the span it declares. That segment was sealed by a writer that + // folded across a hole (the class this file's density law now bars). + // The segment is INTACT and nothing is lost; it simply never held this + // generation. Answering "not packed" is the honest answer, and it lets + // the caller's two-tier read decide what a genuinely absent generation + // means, instead of every compaction pass dying on a repeating throw. + // `actualRanges()` keeps such holes out of committedRanges at open, so + // in a healed store nobody asks this question in the first place. + // + // (2) A DENSE SEGMENT missing a frame it says it has โ€” the manifest and + // the sidecar disagree about a segment that claims to be complete. + // That IS damage, and it stays loud. + if (this.isSparse(meta)) { + prodLog.warn( + `[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` + + `declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` + + `holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` + + `-generation span โ€” it was sealed across a gap and never held this generation. ` + + `Reporting it as unpacked rather than as damage; no record is lost.` + ) + return null + } throw new Error( `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame โ€” packed history is damaged` + `range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + + `span โ€” the manifest and the sidecar disagree; packed history is damaged` ) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index fd052c31..da21dc61 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -96,6 +96,35 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' +/** + * @description Split an ascending list of fold candidates into maximal + * CONTIGUOUS runs โ€” `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`. + * + * A sealed segment declares one dense range `[firstGeneration, + * lastGeneration]`, and every reader treats that range as containment. So a + * batch with a hole in it must never become one segment: it would claim a + * generation it does not hold, and the first read of that hole reports the + * packed history as damaged. One run, one segment โ€” the ranges then describe + * exactly what the segments contain. + * + * @param gens - Fold candidates, strictly ascending by generation. + * @returns One array per contiguous run, in ascending order. Empty in, empty out. + */ +export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] { + const runs: FoldGeneration[][] = [] + let run: FoldGeneration[] = [] + for (const g of gens) { + const prev = run[run.length - 1] + if (prev !== undefined && g.generation !== prev.generation + 1) { + runs.push(run) + run = [] + } + run.push(g) + } + if (run.length > 0) runs.push(run) + return runs +} + /** * @description Phases of the {@link GenerationStore.commitTransaction} commit * protocol at which a test-only fault injector can simulate a process crash. @@ -784,9 +813,15 @@ export class GenerationStore { if (storageSupportsFactLog(this.storage)) { this.segments = new GenerationSegmentStore(this.storage) await this.segments.open() - const packedRanges = this.segments - .segments() - .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) + // ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law + // writer can declare a span wider than the frames it holds; seeding + // committedRanges from the declared span re-admits those holes as + // committed generations, and every later maintenance pass then asks for a + // frame that was never written. `actualRanges()` reads the real + // generation list from the sidecar for exactly those segments (and does + // no I/O for the dense ones, which is all of them on a healthy store). + const packedRanges = (await this.segments.actualRanges()) + .map((r): [number, number] => [r[0], Math.min(r[1], this.committed)]) .filter(([lo, hi]) => lo <= hi) if (packedRanges.length > 0) { // Merge packed (older) + live (newer) interval sets โ€” both ascending; @@ -3121,13 +3156,26 @@ export class GenerationStore { foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) } if (foldInput.length === 0) continue - await segments.fold(foldInput) - segmentsCreated++ - // Segment + manifest durable โ†’ the live copies retire. - for (const g of foldInput) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + // SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous โ€” three + // filters above punch holes in it: a generation missing from + // committedRanges never appears, one still in the pending buffer is + // skipped, and one whose tx.json will not read is skipped. A sealed + // segment declares a DENSE range, so folding across such a hole makes + // the segment claim a generation it does not hold; the next open + // merges that mis-declared range into committedRanges, and every + // subsequent auto-compaction pass then asks for the missing frame and + // fails with "packed history is damaged". Fold each contiguous RUN as + // its own segment instead โ€” same bytes, honest ranges. + for (const run of contiguousRuns(foldInput)) { + if (deadline !== undefined && Date.now() >= deadline) break + await segments.fold(run) + segmentsCreated++ + // Segment + manifest durable โ†’ the live copies retire. + for (const g of run) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + } + folded += run.length } - folded += foldInput.length } if (folded > 0) { prodLog.info( diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts index 2bcee038..bb07268d 100644 --- a/tests/integration/history-repacking.test.ts +++ b/tests/integration/history-repacking.test.ts @@ -16,6 +16,7 @@ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as path from 'node:path' import * as os from 'node:os' +import * as zlib from 'node:zlib' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { GenerationStore } from '../../src/db/generationStore.js' @@ -57,6 +58,107 @@ describe('history repacking โ€” the two-tier lifecycle', () => { } }) + /** + * THE HOLE, END TO END โ€” the shape a real store carries. + * + * A forensic fixture was measured with generation directories 1..2503 + * present except for exactly one: 1416. Its fact-log segment already showed + * the tell โ€” `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration + * 1940 (531 generations) while recording only 530 facts. + * + * Before the fix, repacking such a store folded ACROSS that hole: the batch + * skipped 1416 (no readable delta) and the sealed segment declared a range + * spanning it anyway. The next open merged that declared range back into + * committedRanges, re-admitting 1416 as committed history, and every + * subsequent auto-compaction pass then asked the packed tier for a frame + * that was never written โ€” producing, on EVERY run, the non-fatal narration + * + * Auto-compaction of generational history failed (non-fatal): generation + * N is inside sealed segment seg-....bgs's declared range but has no frame + * โ€” packed history is damaged + * + * This pin removes a generation directory to make the same hole, then + * requires repack + reopen + compaction to complete cleanly. + */ + it('a missing generation directory does not poison the packed tier', async () => { + const dir = tempDir() + // `retention: 'all'` throughout: close() otherwise auto-compacts the + // history away, and this pin needs the cold generations still on disk so + // there is something to punch a hole in. The live window stays at its + // production default for the build phase, so nothing folds yet. + const archival = async (): Promise => { + const b = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + embeddingFunction: stub, + retention: 'all' + }) + await b.init() + return b + } + const brain = await archival() + + const id = await brain.add({ + data: 'holed-entity', + type: NounType.Document, + metadata: { v: 0 } + }) + // One flush per update: single-op writes coalesce inside a flush window, + // so a history deep enough to have a middle needs the windows separated. + for (let v = 1; v <= 12; v++) { + await brain.update({ id, metadata: { v } }) + await brain.flush() + } + await brain.close() + + // Punch the hole: delete ONE generation directory in the middle of the + // cold range, exactly as the real store presents it. + const genRoot = path.join(dir, '_generations') + const numeric = fs + .readdirSync(genRoot, { withFileTypes: true }) + .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)) + .map((e) => Number(e.name)) + .sort((a, b) => a - b) + expect(numeric.length).toBeGreaterThan(6) + const victim = numeric[Math.floor(numeric.length / 2)] + fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true }) + + // Now shrink the live window and reopen. close() repacks automatically + // (brainy.ts phase 0b), so this is the production sequence exactly: a + // store with a hole in its history gets folded by ordinary housekeeping, + // with nobody asking for it. + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 + const reopened = await archival() + const result = await reopened.repackHistory() + expect(result.foldedGenerations).toBeGreaterThan(0) + + const segDir = path.join(dir, SEGMENTS_PREFIX) + const manifestPath = ['manifest.json', 'manifest.json.gz'] + .map((f) => path.join(segDir, f)) + .find((p) => fs.existsSync(p))! + const raw = manifestPath.endsWith('.gz') + ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8') + : fs.readFileSync(manifestPath, 'utf8') + const manifest = JSON.parse(raw) as { + segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }> + } + + // THE LAW: every sealed segment declares exactly as many generations as it + // holds frames, and none of them spans the victim. + for (const s of manifest.segments) { + expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames) + expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false) + } + + await reopened.close() + + // And the pass that used to fail on every run now completes: reopen (which + // re-seeds committedRanges from the packed tier) then compact history. + const third = await openBrain(dir) + await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined() + await third.close() + }) + it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 const dir = tempDir() diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts index 27ab85cb..f16e67b3 100644 --- a/tests/unit/db/generation-segments.test.ts +++ b/tests/unit/db/generation-segments.test.ts @@ -147,4 +147,119 @@ describe('db/GenerationSegmentStore โ€” the D1+D3 packed tier', () => { await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) + + // ========================================================================== + // THE DENSITY LAW + // ========================================================================== + // + // A sealed segment declares a CONTIGUOUS range and every reader treats that + // range as containment. Folding a sparse batch therefore makes the segment + // claim generations it does not hold โ€” and because `open()` merges declared + // ranges back into committedRanges, the hole is re-admitted as committed + // history and every later maintenance pass fails asking for a frame that was + // never written. That is the "generation N is inside sealed segment + // seg-....bgs's declared range but has no frame โ€” packed history is damaged" + // narration seen on every run of the affected stores. + + it('fold REFUSES a batch with a hole โ€” a dense range may not be declared over sparse input', async () => { + await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow( + /not contiguous: 2 โ†’ 4 skips 1 generation/ + ) + // The refusal loses nothing: no segment was sealed, so the generations + // stay in the live tier and the next pass folds them correctly. + expect(store.segments()).toHaveLength(0) + expect(store.hasGeneration(1)).toBe(false) + }) + + it('a wider gap names how many generations it would have swallowed', async () => { + await expect(store.fold([gen(10), gen(20)])).rejects.toThrow( + /not contiguous: 10 โ†’ 20 skips 9 generation\(s\)/ + ) + }) + + it('two contiguous runs folded separately declare honest ranges', async () => { + // What the caller now does instead of folding across the gap. + const a = await store.fold([gen(1), gen(2), gen(3)]) + const b = await store.fold([gen(7), gen(8)]) + expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) + expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 }) + // The gap is honestly outside the packed tier. + for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false) + for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true) + expect(await store.actualRanges()).toEqual([ + [1, 3], + [7, 8] + ]) + }) + + it('actualRanges() is exact and I/O-free for dense segments', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + // Adjacent dense segments each contribute their declared range. + expect(await store.actualRanges()).toEqual([ + [1, 2], + [3, 4] + ]) + }) + + // ---- pre-existing damage: a store sealed by the old writer ---------------- + + /** + * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and + * sidecar for a contiguous run, then rewrite the manifest so the segment + * declares a wider range than the frames it holds. This reproduces on disk + * exactly what the affected stores carry, without needing the old code. + */ + const sealSparseSegment = async (): Promise => { + await store.fold([gen(1), gen(2), gen(3)]) + const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any + // Declare 1..5 while holding frames for 1..3 โ€” generations 4 and 5 become + // holes inside a sealed range. + manifest.segments[0].lastGeneration = 5 + await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest) + } + + it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + + // The frames it really holds still serve, byte-faithfully. + expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002) + expect(await reopened.readRecords(3)).toHaveLength(2) + + // The holes answer "not packed" instead of throwing. This is the fix for + // the wedge: the old reader threw here on EVERY maintenance pass. + expect(await reopened.readDelta(4)).toBeNull() + expect(await reopened.readRecords(5)).toBeNull() + }) + + it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + // Declared 1..5; actually holds 1..3. The store seeds committedRanges from + // THIS, so generations 4 and 5 never become committed history again. + expect(await reopened.actualRanges()).toEqual([[1, 3]]) + }) + + it('a DENSE segment missing a frame is still loud damage', async () => { + // The other side of the branch: when the manifest claims a complete span, + // a missing frame means the manifest and sidecar disagree โ€” real damage, + // and it must not be quietly downgraded to "unpacked". + await store.fold([gen(1), gen(2), gen(3)]) + const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` + const raw = (await storage.readRawBytes(idxPath))! + const { decode, encode } = await import('@msgpack/msgpack') + const idx = decode(raw) as any + // Drop generation 2's entry while the manifest still declares 3 frames. + idx.generations = idx.generations.filter(([g]: [number]) => g !== 2) + await storage.writeRawBytes(idxPath, encode(idx)) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + await expect(reopened.readDelta(2)).rejects.toThrow( + /manifest and the sidecar disagree; packed history is damaged/ + ) + }) }) From d6bcb14f698de40507a6ce16daf56ab235f71924 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:30:46 -0700 Subject: [PATCH 07/24] =?UTF-8?q?build(release):=20the=20docs-push=20step?= =?UTF-8?q?=20retires=20=E2=80=94=20this=20engine=20documents=20itself=20i?= =?UTF-8?q?n=20its=20own=20repository?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one-doc-set ruling (2026-08-31) gives soulcraft.com/docs to the paid product alone; the site serves redirects for the slugs this rail used to push. The push script stays in the tree as history; the rail stops calling it. (cherry picked from commit 655aa13ea79e23927cde7fd47ab13b505cb042d9) --- scripts/release.sh | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 08293e3a..1a4fe575 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -248,17 +248,12 @@ else echo -e "${RED}โš ๏ธ FORGEJO_RELEASE_TOKEN unset โ€” no release page created; tag + CHANGELOG remain the record${NC}\n" fi -# Step 12: Push public docs to the soulcraft.com docs ingest door -# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when -# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish โ€” -# that already happened) when a push errors, so the docs site never -# silently trails npm. -echo -e "${BLUE}1๏ธโƒฃ2๏ธโƒฃ Pushing public docs to soulcraft.com/docs...${NC}" -if node scripts/push-docs.js; then - echo -e "${GREEN}โœ… Docs push step done${NC}\n" -else - echo -e "${RED}โŒ Docs push FAILED โ€” soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" -fi +# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): +# soulcraft.com/docs carries the paid product's documentation only. This +# engine's documentation home is THIS repository โ€” README and docs/ โ€” and the +# site serves 301s for the slugs this rail used to push. The push script stays +# in the tree for history; the rail no longer calls it. +echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" echo -e "${GREEN}โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”${NC}" echo -e "${GREEN}๐ŸŽ‰ Release ${NEW_VERSION} complete!${NC}" From 0f0022b1c9abd710184d0e6ac573b6a07d8ec068 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 12:34:36 -0700 Subject: [PATCH 08/24] chore(release): 10.4.5 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a54d609e..6c05eba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31) + +- build(release): the docs-push step retires โ€” this engine documents itself in its own repository (d6bcb14f) +- fix(generations): a sealed segment may only declare the generations it holds (a963a744) +- fix(recovery): a torn generation-log tail is a terminal verdict, never a wait (c9930871) + + ### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) - fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) diff --git a/package-lock.json b/package-lock.json index c4f66561..2528227b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.5", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 06ce0253..31448825 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.5", "brainyContract": 1, "description": "Universal Knowledge Protocolโ„ข - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns ร— 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From 73500e7d109275570857341000c9bad44d5cc1f3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 12:59:40 -0700 Subject: [PATCH 09/24] fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit transact()'s delete legs (direct unrelate and the noun-remove cascade) hand the SAME verb object to the graph-retraction op and the metadata-retraction op. The metadata leg sanitized at PLAN time, when the verb was still clean, so the wrap returned the same reference โ€” then the graph op's execute-time endpoint resolution (deliberately deferred for same-batch forward refs) mirrored BigInt sourceInt/targetInt onto the shared object, and the metadata op crossed the seam with them. A strict provider rightly refuses that crossing, so every transact-wrapped edge delete aborted; direct unrelate() resolves ints at build time, before its sanitize, which is why no existing gate saw it. The JSON-safe view now lives in a shared leaf (utils/jsonSafeIndexMetadata) and is applied INSIDE AddToMetadataIndexOperation and RemoveFromMetadataIndexOperation at execute and rollback time โ€” the one place no plan-vs-execute ordering can bypass. Pins: the fleet repro, the cascade shape, a mixed batch, and unit pins that mutate the entity after construction against a strict seam (5 red before, 5 green after). --- src/brainy.ts | 30 +-- src/transaction/operations/IndexOperations.ts | 29 ++- src/utils/jsonSafeIndexMetadata.ts | 47 +++++ ...ansact-edge-delete-bigint-aliasing.test.ts | 184 ++++++++++++++++++ 4 files changed, 263 insertions(+), 27 deletions(-) create mode 100644 src/utils/jsonSafeIndexMetadata.ts create mode 100644 tests/integration/transact-edge-delete-bigint-aliasing.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index da04577e..06c947c2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -15,6 +15,7 @@ import { JsHnswVectorIndex } from './hnsw/hnswIndex.js' import { createStorage, resolveFilesystemRoot } from './storage/storageFactory.js' import type { StorageOptions } from './storage/storageFactory.js' import { rebuildCounts } from './utils/rebuildCounts.js' +import { jsonSafeIndexMetadata } from './utils/jsonSafeIndexMetadata.js' import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js' import { BaseStorage } from './storage/baseStorage.js' import { @@ -4203,32 +4204,19 @@ export class Brainy implements BrainyInterface { */ /** * @description A JSON-safe view of a record bound for the metadata-index - * crossing. The seam's metadata is JSON-safe BY CONTRACT (a native provider - * serializes it; u64 ints as Number corrupt above 2^53) โ€” but - * {@link resolveVerbEndpointInts} MIRRORS the resolved endpoint ints onto - * the verb object itself as BigInt (`verb.sourceInt`/`targetInt`), so a - * verb object reused as index metadata carried BigInts into - * JSON.stringify, which throws, aborting the whole transaction (found by - * the first joint pair gate). Endpoint ints ride their OWN op params on the - * graph legs โ€” the metadata crossing drops every BigInt-valued top-level - * key instead of guessing at a lossy numeric encoding. + * crossing โ€” delegates to the shared {@link jsonSafeIndexMetadata} leaf, + * which the metadata-index transaction operations ALSO apply at execute + * and rollback time. This plan-time wrap alone proved insufficient: it + * returns the same reference when the record is clean, and `transact()`'s + * delete legs share that reference with a graph-retraction op whose + * execute-time endpoint resolution mirrors BigInt ints onto it (the full + * aliasing story lives on the leaf module's doc). * @param metadata - The candidate index-metadata record. * @returns The same object when already JSON-safe, else a shallow copy * without the BigInt-valued keys. */ private static jsonSafeIndexMetadata(metadata: unknown): unknown { - if (metadata === null || typeof metadata !== 'object') return metadata - const rec = metadata as Record - let hasBigint = false - for (const k in rec) { - if (typeof rec[k] === 'bigint') { hasBigint = true; break } - } - if (!hasBigint) return metadata - const out: Record = {} - for (const k in rec) { - if (typeof rec[k] !== 'bigint') out[k] = rec[k] - } - return out + return jsonSafeIndexMetadata(metadata) } private metadataIndexRetractionOp( diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 1bbbca88..0142dc54 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -14,6 +14,7 @@ import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' import { isZeroNormVector } from '../../utils/distance.js' +import { jsonSafeIndexMetadata } from '../../utils/jsonSafeIndexMetadata.js' import { prodLog } from '../../utils/logger.js' /** @@ -390,13 +391,21 @@ export class AddToMetadataIndexOperation implements Operation { // rollback so add + undo reference the same watermark. const generation = this.generationFn?.() - // Add to metadata index (skipFlush=true for transaction atomicity) - await this.index.addToIndex(this.id, this.entity, true, false, generation) + // The JSON-safe view is taken HERE, per crossing, never at construction: + // the entity reference this op holds can be mutated between plan and + // execute (a graph op's execute-time endpoint-int resolution mirrors + // BigInts onto a shared verb object) โ€” see jsonSafeIndexMetadata's + // module doc. + await this.index.addToIndex( + this.id, jsonSafeIndexMetadata(this.entity), true, false, generation + ) // Return rollback action return async () => { // Remove from metadata index - await this.index.removeFromIndex(this.id, this.entity, generation) + await this.index.removeFromIndex( + this.id, jsonSafeIndexMetadata(this.entity), generation + ) } } } @@ -432,13 +441,21 @@ export class RemoveFromMetadataIndexOperation implements Operation { // Resolve the removal generation once; reuse it for the rollback re-add. const generation = this.generationFn?.() - // Remove from metadata index - await this.index.removeFromIndex(this.id, this.entity, generation) + // Sanitized per crossing, never at construction โ€” transact()'s delete + // legs hand this op the SAME verb object the graph-retraction op's + // execute-time endpoint resolution mutates (BigInt sourceInt/targetInt), + // so a plan-time view aliases the pollution. See jsonSafeIndexMetadata's + // module doc. + await this.index.removeFromIndex( + this.id, jsonSafeIndexMetadata(this.entity), generation + ) // Return rollback action return async () => { // Re-add with original metadata (skipFlush=true) - await this.index.addToIndex(this.id, this.entity, true, false, generation) + await this.index.addToIndex( + this.id, jsonSafeIndexMetadata(this.entity), true, false, generation + ) } } } diff --git a/src/utils/jsonSafeIndexMetadata.ts b/src/utils/jsonSafeIndexMetadata.ts new file mode 100644 index 00000000..d3b1be5f --- /dev/null +++ b/src/utils/jsonSafeIndexMetadata.ts @@ -0,0 +1,47 @@ +/** + * @module utils/jsonSafeIndexMetadata + * @description The metadata-index crossing's JSON-safety law, as a leaf + * function both the coordinator and the transaction operations share. + * + * The seam's metadata is JSON-safe BY CONTRACT (a native provider serializes + * it; u64 ints as Number corrupt above 2^53) โ€” but `resolveVerbEndpointInts` + * MIRRORS the resolved endpoint ints onto the verb object itself as BigInt + * (`verb.sourceInt`/`targetInt`), so a verb object reused as index metadata + * carries BigInts into JSON.stringify, which throws, aborting the whole + * transaction. Endpoint ints ride their OWN op params on the graph legs โ€” the + * metadata crossing drops every BigInt-valued top-level key instead of + * guessing at a lossy numeric encoding. + * + * WHY THIS IS A LEAF MODULE, ENFORCED AT THE CROSSING: sanitizing only at + * operation-construction time is not enough. `transact()`'s delete legs pass + * the SAME verb object to both the graph-retraction op (whose endpoint-int + * thunk deliberately resolves at EXECUTE time, for same-batch forward refs) + * and the metadata-retraction op. At plan time the verb is still clean, so a + * plan-time sanitize returns the same reference โ€” then the graph op executes + * first, mirrors the BigInt ints onto the shared object, and the metadata op + * crosses the seam with them (found by the first fleet adoption of the native + * pair: every transact-wrapped edge delete aborted). The crossing itself is + * the only place ordering cannot bypass. + */ + +/** + * A JSON-safe view of a record bound for the metadata-index crossing. + * + * @param metadata - The candidate index-metadata record. + * @returns The same object when already JSON-safe, else a shallow copy + * without the BigInt-valued keys. + */ +export function jsonSafeIndexMetadata(metadata: unknown): unknown { + if (metadata === null || typeof metadata !== 'object') return metadata + const rec = metadata as Record + let hasBigint = false + for (const k in rec) { + if (typeof rec[k] === 'bigint') { hasBigint = true; break } + } + if (!hasBigint) return metadata + const out: Record = {} + for (const k in rec) { + if (typeof rec[k] !== 'bigint') out[k] = rec[k] + } + return out +} diff --git a/tests/integration/transact-edge-delete-bigint-aliasing.test.ts b/tests/integration/transact-edge-delete-bigint-aliasing.test.ts new file mode 100644 index 00000000..b3902571 --- /dev/null +++ b/tests/integration/transact-edge-delete-bigint-aliasing.test.ts @@ -0,0 +1,184 @@ +/** + * @module tests/integration/transact-edge-delete-bigint-aliasing + * @description Regression for a fleet-adoption blocker: ANY edge delete + * inside `transact()` โ€” a direct unrelate or a noun-remove's cascade โ€” + * aborted with the metadata seam's BigInt JSON-guard error on a strict + * (native) metadata provider. + * + * The aliasing chain: `planTxUnrelate`/the remove-cascade pass the SAME verb + * object to the graph-retraction op and the metadata-retraction op. The + * metadata leg's JSON-safe wrap ran at PLAN time, when the verb was still + * clean โ€” so it returned the same reference. At EXECUTE time the graph op + * runs first and `resolveVerbEndpointInts` mirrors BigInt + * `sourceInt`/`targetInt` onto the shared object (deliberately deferred for + * same-batch forward refs โ€” see transact-forward-ref-graph.test.ts); the + * metadata op then crossed the seam with the polluted object. Direct + * `unrelate()` resolves ints at BUILD time, before its sanitize, which is why + * only the transact() shapes ever hit it. + * + * Fix under pin: the JSON-safe view is taken AT THE CROSSING โ€” inside the + * metadata-index operations' execute/rollback โ€” so no plan-vs-execute + * ordering can bypass it. The JS baseline index tolerates BigInts (it would + * mask the bug), so these pins SPY on the seam and assert what actually + * crossed, exactly as a strict native provider would judge it. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { + AddToMetadataIndexOperation, + RemoveFromMetadataIndexOperation +} from '../../src/transaction/operations/index.js' + +let seq = 0 +const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + +/** Top-level BigInt-valued keys of a candidate seam crossing (the guard's law). */ +const bigintKeys = (metadata: unknown): string[] => { + if (metadata === null || typeof metadata !== 'object') return [] + return Object.entries(metadata as Record) + .filter(([, v]) => typeof v === 'bigint') + .map(([k]) => k) +} + +describe('transact() edge deletes never carry BigInt across the metadata seam', () => { + let dir: string + let brain: any + let crossings: Array<{ door: string; id: string; keys: string[] }> + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-tx-bigint-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + + // Spy on the seam the way a strict native provider judges it: record the + // BigInt-valued top-level keys of every metadata argument that crosses. + // The JS baseline index tolerates BigInts, so without this the baseline + // run would green a shape the native pair aborts on. + crossings = [] + const index = brain.metadataIndex + for (const door of ['addToIndex', 'removeFromIndex'] as const) { + const real = index[door].bind(index) + index[door] = (id: string, metadata: unknown, ...rest: unknown[]) => { + crossings.push({ door, id, keys: bigintKeys(metadata) }) + return real(id, metadata, ...rest) + } + } + }) + + afterEach(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('CASE 1 (the fleet repro): relate, then transact([{op: unrelate}])', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const verbId = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + + crossings.length = 0 + await brain.transact([{ op: 'unrelate', id: verbId }]) + + const polluted = crossings.filter((c) => c.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.storage.getVerb(verbId)).toBeFalsy() + }) + + it('CASE 2 (the cascade shape): transact([{op: remove}]) cascading edge deletes', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const c = await brain.add({ id: freshId(), data: 'c', type: NounType.Thing }) + const ab = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + const ca = await brain.relate({ from: c, to: a, type: VerbType.RelatedTo }) + + crossings.length = 0 + await brain.transact([{ op: 'remove', id: a }]) + + const polluted = crossings.filter((c2) => c2.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.get(a)).toBeFalsy() + expect(await brain.storage.getVerb(ab)).toBeFalsy() + expect(await brain.storage.getVerb(ca)).toBeFalsy() + }) + + it('CASE 3 (one batch, both legs): adds + relate + unrelate of a pre-existing edge', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const old = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + + const x = freshId() + crossings.length = 0 + await brain.transact([ + { op: 'add', id: x, data: 'x', type: NounType.Thing }, + { op: 'relate', from: a, to: x, type: VerbType.RelatedTo }, + { op: 'unrelate', id: old } + ]) + + const polluted = crossings.filter((c) => c.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.storage.getVerb(old)).toBeFalsy() + const edges = await brain.related({ from: a }) + expect(edges.length).toBe(1) + expect(edges[0].id).not.toBe(old) + }) +}) + +describe('the metadata-index operations sanitize at the crossing, not at construction', () => { + /** A strict seam: refuses BigInts exactly as the native provider does. */ + const strictIndex = () => { + const seen: Array<{ door: string; keys: string[] }> = [] + const judge = (door: string, metadata: unknown) => { + const keys = bigintKeys(metadata) + seen.push({ door, keys }) + if (keys.length > 0) { + throw new Error( + `${door}: the metadata object violates the provider seam's JSON ` + + `contract โ€” BigInt at ${keys.join(', ')}.` + ) + } + } + return { + seen, + addToIndex: async (_id: string, metadata: unknown) => judge('addToIndex', metadata), + removeFromIndex: async (_id: string, metadata: unknown) => judge('removeFromIndex', metadata) + } + } + + it('RemoveFromMetadataIndexOperation: entity mutated AFTER construction still crosses clean', async () => { + const index = strictIndex() + const verb: Record = { id: 'v1', sourceId: 'a', targetId: 'b' } + const op = new RemoveFromMetadataIndexOperation(index as any, 'v1', verb, () => 7n) + + // The graph leg's execute-time endpoint resolution, simulated: the shared + // object is polluted between plan and execute. + verb.sourceInt = 800_000n + verb.targetInt = 800_001n + + const rollback = await op.execute() + await rollback() + expect(index.seen.map((s) => s.keys)).toEqual([[], []]) + }) + + it('AddToMetadataIndexOperation: same law on the add leg and its rollback', async () => { + const index = strictIndex() + const verb: Record = { id: 'v2', sourceId: 'a', targetId: 'b' } + const op = new AddToMetadataIndexOperation(index as any, 'v2', verb, () => 7n) + + verb.sourceInt = 800_000n + verb.targetInt = 800_001n + + const rollback = await op.execute() + await rollback() + expect(index.seen.map((s) => s.keys)).toEqual([[], []]) + }) +}) From 4014e0f12593f9c781dde0bfcf1a93b0c90d71cc Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 14:50:45 -0700 Subject: [PATCH 10/24] chore(release): 10.4.6 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c05eba6..7154d5a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31) + +- fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d) + + ### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31) - build(release): the docs-push step retires โ€” this engine documents itself in its own repository (d6bcb14f) diff --git a/package-lock.json b/package-lock.json index 2528227b..9e573da3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.5", + "version": "10.4.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.5", + "version": "10.4.6", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 31448825..51322998 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.5", + "version": "10.4.6", "brainyContract": 1, "description": "Universal Knowledge Protocolโ„ข - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns ร— 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From e64e2bc17580737a0b7a63e2af8ea6b6c527278a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 12:04:29 -0700 Subject: [PATCH 11/24] =?UTF-8?q?docs(releases):=20the=20release-notes=20d?= =?UTF-8?q?oor=20=E2=80=94=20owner-language=20notes=20for=20both=20engines?= =?UTF-8?q?,=20backfilled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fleet's releases wall reads one public URL per product. These files are that door for Brainy and Open Brainy: newest first, honest history from the changelog, one entry appended by every release from here on. --- releases/brainy.json | 52 +++++++++++++++++++++++ releases/open-brainy.json | 87 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 releases/brainy.json create mode 100644 releases/open-brainy.json diff --git a/releases/brainy.json b/releases/brainy.json new file mode 100644 index 00000000..17217a6e --- /dev/null +++ b/releases/brainy.json @@ -0,0 +1,52 @@ +{ + "product": "brainy", + "entries": [ + { + "version": "11.0.3", + "date": "2026-09-01", + "headline": "The embedding upgrade ceremony runs on every brain", + "items": [ + "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", + "A one-fix release; nothing else changed." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.2", + "date": "2026-08-31", + "headline": "One embedding quality everywhere, 3โ€“4ร— faster imports", + "items": [ + "Every runtime embeds with the same full-precision model โ€” search quality no longer depends on where you run.", + "Bulk embedding measured 3.1โ€“4.2ร— faster, and an online re-embed ceremony upgrades existing stores without downtime.", + "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.1", + "date": "2026-08-31", + "headline": "Deletes inside transactions are safe", + "items": [ + "Deleting relations inside a transact() no longer corrupts index bookkeeping.", + "A store that deletes its last relation keeps serving instead of refusing." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.0", + "date": "2026-08-28", + "headline": "One install, one engine โ€” Brainy", + "items": [ + "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", + "A missing native build refuses loudly with its cures named; nothing falls back silently.", + "Stores open in place โ€” no migration." + ], + "url": null, + "thumb": null + } + ], + "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." +} diff --git a/releases/open-brainy.json b/releases/open-brainy.json new file mode 100644 index 00000000..21014f5b --- /dev/null +++ b/releases/open-brainy.json @@ -0,0 +1,87 @@ +{ + "product": "open-brainy", + "entries": [ + { + "version": "10.4.6", + "date": "2026-08-31", + "headline": "Transactions cross the index seam safely", + "items": [ + "Deleting relations inside a transact() no longer fails against the metadata index โ€” operations take a JSON-safe view at the moment they execute.", + "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", + "thumb": null + }, + { + "version": "10.4.5", + "date": "2026-08-31", + "headline": "Recovery tells the truth, docs live at home", + "items": [ + "A torn generation-log tail is a terminal verdict with a named cure โ€” never an endless wait at open.", + "A sealed segment declares only the generations it actually holds.", + "The engine's documentation now publishes from its own repository." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", + "thumb": null + }, + { + "version": "10.4.4", + "date": "2026-08-28", + "headline": "Faster opens, quieter idle", + "items": [ + "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", + "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", + "A slow open now names the exact step it is in, so operators see what is being paid and why." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", + "thumb": null + }, + { + "version": "10.4.3", + "date": "2026-08-27", + "headline": "Open Brainy, under its own name", + "items": [ + "The same engine as 10.4.2, now published as @soulcraftlabs/brainy โ€” the MIT reference engine, on The Source.", + "No code changes; your imports change once and everything else stays put." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", + "thumb": null + }, + { + "version": "10.4.2", + "date": "2026-08-27", + "headline": "Vectors that lie are refused, counts that drift are caught", + "items": [ + "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", + "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", + "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", + "thumb": null + }, + { + "version": "10.4.1", + "date": "2026-08-26", + "headline": "Writes that change nothing cost nothing", + "items": [ + "The read gate is per index family, and a write carrying unchanged data never re-embeds.", + "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", + "thumb": null + }, + { + "version": "10.4.0", + "date": "2026-08-26", + "headline": "Repair routing, the vector ledger, and honest empties", + "items": [ + "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", + "An empty string is real data, not a missing field.", + "The metadata crossing never carries raw integer relation endpoints โ€” a whole class of serialization faults closed." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", + "thumb": null + } + ], + "history": "Earlier releases are recorded in CHANGELOG.md in this repository." +} From f097cbf6f29cad03a34ca0c31c64fd86b53c9fdb Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 13:49:26 -0700 Subject: [PATCH 12/24] =?UTF-8?q?docs(releases):=20the=2010.4.7=20note=20?= =?UTF-8?q?=E2=80=94=20count=20ledgers=20can=20no=20longer=20race=20themse?= =?UTF-8?q?lves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/open-brainy.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index 21014f5b..e1dc83ce 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,17 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.7", + "date": "2026-09-01", + "headline": "Count ledgers can no longer race themselves", + "items": [ + "Concurrent count flushes coalesce into one writer with a trailing pass โ€” parallel flushes can no longer corrupt a store's count ledger.", + "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", + "thumb": null + }, { "version": "10.4.6", "date": "2026-08-31", From 7ab670b525d62c86d7a39f81acdb1384fe2928ba Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 13:55:27 -0700 Subject: [PATCH 13/24] =?UTF-8?q?docs(releases):=20the=2011.0.4=20note=20?= =?UTF-8?q?=E2=80=94=20millisecond=20closes,=20storm-free=20rebuilds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/brainy.json b/releases/brainy.json index 17217a6e..c6c664a9 100644 --- a/releases/brainy.json +++ b/releases/brainy.json @@ -1,6 +1,18 @@ { "product": "brainy", "entries": [ + { + "version": "11.0.4", + "date": "2026-09-01", + "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", + "items": [ + "close() no longer pays deferred compaction or waits out an in-flight rebuild โ€” measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", + "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step โ€” the same guarantee, a fraction of the disk traffic.", + "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." + ], + "url": null, + "thumb": null + }, { "version": "11.0.3", "date": "2026-09-01", From 297a3d76575771f60518a813b2c023e68bc9d707 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 08:30:01 -0700 Subject: [PATCH 14/24] =?UTF-8?q?docs(releases):=20the=2010.4.9=20note=20?= =?UTF-8?q?=E2=80=94=20graph-first=20finds,=20honest=20verb=20arrays,=20bo?= =?UTF-8?q?unded=20recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/open-brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index e1dc83ce..582f4847 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,18 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.9", + "date": "2026-09-02", + "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", + "items": [ + "find({ connected, where }) now walks the neighbours first and filters only those rows โ€” correct at every page, and O(neighbours) instead of O(store).", + "related() with a list of verb types (or sources, or targets) returns every requested kind โ€” four fast paths silently kept only the first.", + "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open โ€” measured at two minutes on a large brain, now milliseconds." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", + "thumb": null + }, { "version": "10.4.7", "date": "2026-09-01", From 4f1e27c9a089f5dc5d3b20f7ba9fc52384ee0e28 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 08:32:21 -0700 Subject: [PATCH 15/24] =?UTF-8?q?docs(releases):=20the=2011.0.5=20note=20?= =?UTF-8?q?=E2=80=94=20graph-first=20finds=20in=20production,=20bounded=20?= =?UTF-8?q?recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/brainy.json b/releases/brainy.json index c6c664a9..8f61c7f2 100644 --- a/releases/brainy.json +++ b/releases/brainy.json @@ -1,6 +1,18 @@ { "product": "brainy", "entries": [ + { + "version": "11.0.5", + "date": "2026-09-02", + "headline": "Graph-first finds in production, and opens that stop rescanning history", + "items": [ + "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door โ€” correct at every page and O(neighbours), never the whole store.", + "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", + "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open โ€” measured at two minutes on a large brain, now milliseconds." + ], + "url": null, + "thumb": null + }, { "version": "11.0.4", "date": "2026-09-01", From 3dadbec8f21574dab4ce72919576769ff209e7e1 Mon Sep 17 00:00:00 2001 From: Fleet Bot Date: Wed, 2 Sep 2026 20:56:24 +0200 Subject: [PATCH 16/24] ci: superseded pushes cancel their own runs (concurrency per ref) --- .forgejo/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 5e93cd96..da5887f6 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -5,6 +5,10 @@ name: CI # sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the # tag's publish-source run and starve every release (observed on 8.10.3 and # 9.0.0: the publish sat behind the tag's own redundant CI). +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + on: push: branches: ['**'] From 08758c254fe04c7f84cadf930540aebf0f8b8093 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 12:16:42 -0700 Subject: [PATCH 17/24] =?UTF-8?q?docs(releases):=20the=2010.4.10=20note=20?= =?UTF-8?q?=E2=80=94=20a=20planner=20door,=20batched=20containment=20repai?= =?UTF-8?q?r,=20a=20fixed=20near()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate: 10.4.10 candidate (a8c5fbf9) vs 10.4.9 control (eec90bdd) โ€” collected 3,223/3,211, 0 new reds. shasum ffff79c5c4bcbc614545ad72e8d0138c039062e9. --- releases/open-brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index 582f4847..dab25971 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,18 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.10", + "date": "2026-09-02", + "headline": "A planner door for indexes, batched containment repair, and a fixed near()", + "items": [ + "An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.", + "repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.", + "find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10", + "thumb": null + }, { "version": "10.4.9", "date": "2026-09-02", From 3835a0e7027bd215bdbb75d4e4795195981bd03f Mon Sep 17 00:00:00 2001 From: Fleet Bot Date: Wed, 2 Sep 2026 22:51:59 +0200 Subject: [PATCH 18/24] =?UTF-8?q?ci(publish):=20allow=20manual=20dispatch?= =?UTF-8?q?=20=E2=80=94=20replay=20lane=20for=20dropped=20tag=20events?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .forgejo/workflows/publish-source.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml index 58cb1d30..6bd42b2a 100644 --- a/.forgejo/workflows/publish-source.yml +++ b/.forgejo/workflows/publish-source.yml @@ -12,6 +12,11 @@ on: push: tags: - 'v*' + workflow_dispatch: + inputs: + ref_reason: + description: 'why this manual run (e.g. tag event dropped)' + required: false jobs: publish: From 61bc5f423b208794262126270e46ffe4e3230689 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:03:28 -0700 Subject: [PATCH 19/24] =?UTF-8?q?docs(releases):=20the=2010.4.11=20note=20?= =?UTF-8?q?=E2=80=94=20hybrid=20filter-before-hydrate,=20one=20shutdown=20?= =?UTF-8?q?owner,=20a=20faster=20open?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate: final tip 27759a1b vs a8c5fbf9 (10.4.10) control โ€” collected 3,212, 0 new reds after two fix cycles (coverage-guard registration + perf-lane classification; a real budget flake in the batch-size test switched to unvectored items). shasum ffc33df95b2709dfcc8c67ac961991e3153f8883. --- releases/open-brainy.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index dab25971..9f1cd239 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,20 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.11", + "date": "2026-09-02", + "headline": "Hybrid finds filter before they hydrate, one owner per shutdown, and a faster open", + "items": [ + "Hybrid finds (query/vector combined with a filter, including connected and fusion finds) now filter first and hydrate only the page โ€” one batchGet of exactly the requested rows, instead of hydrating everything the search side found. Fixes a bug where any page after the first came back empty.", + "A brain now has exactly one shutdown owner โ€” a host and its engine no longer race to close the same store, and a follow-up flush requested during a running flush is handed off cleanly instead of ever risking a stall.", + "find({ path }) and other path-scoped VFS searches now serve a real range over the indexed path (O(log n)) instead of refusing the query outright โ€” both scoped and recursive:false searches were silently broken before this.", + "Open no longer rescans a brain's whole fact log on every open โ€” sealed segments the manifest already accounts for are skipped, collapsing a multi-second open term to near-zero on large brains.", + "commitTransaction() now refuses by name if single-ops are still pending, and a read-only open no longer writes clean-shutdown evidence it didn't earn โ€” two correctness invariants that were previously assumed, not enforced." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11", + "thumb": null + }, { "version": "10.4.10", "date": "2026-09-02", From 8752f11f4d5e312a47dde521c267e9b395de0d21 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:13:40 -0700 Subject: [PATCH 20/24] chore(releases): the product engine's release wall leaves the reference repo Only the open engine's own wall (releases/open-brainy.json) belongs in the public reference project. The product's notes are served from the product's own repository. --- releases/brainy.json | 76 -------------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 releases/brainy.json diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door โ€” correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open โ€” measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild โ€” measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step โ€” the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3โ€“4ร— faster imports", - "items": [ - "Every runtime embeds with the same full-precision model โ€” search quality no longer depends on where you run.", - "Bulk embedding measured 3.1โ€“4.2ร— faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine โ€” Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place โ€” no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} From 85b1fa5c1a82ccad2f5ce9bd86b31fc18ab13357 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:15:39 -0700 Subject: [PATCH 21/24] =?UTF-8?q?ci(release):=20mechanize=20the=20releases?= =?UTF-8?q?-wall=20entry=20=E2=80=94=20never=20hand-written=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every release used to get its releases/open-brainy.json entry typed by hand after the fact. scripts/wall-entry.mjs derives it from the CHANGELOG entry release.sh just composed (headline = first bullet, items = every bullet, hash stripped) and prepends it, refusing by name on a duplicate version and validating the whole file's shape + newest-first ordering before and after it writes. release.sh now runs it as its own step, between the CHANGELOG update and the release commit, and stages releases/open-brainy.json into that commit. The product engine's rail runs this identical script against its own releases/brainy.json, unchanged โ€” each repo's wall file lives beside the CHANGELOG it derives from; there is no cross-repo step. A --check mode validates a wall file's exact key set, field types, and newest-first ordering with no duplicates, read-only. tests/unit/release/wall-entry.test.ts covers derivation, prepend, duplicate refusal, and --check's shape/ordering checks over temp copies โ€” never the real files. --check also runs green against both releases/open-brainy.json and releases/brainy.json as they stand today. --- scripts/release.sh | 13 +- scripts/wall-entry.mjs | 364 ++++++++++++++++++++++++++ tests/unit/release/wall-entry.test.ts | 216 +++++++++++++++ 3 files changed, 591 insertions(+), 2 deletions(-) create mode 100644 scripts/wall-entry.mjs create mode 100644 tests/unit/release/wall-entry.test.ts diff --git a/scripts/release.sh b/scripts/release.sh index 5d434320..07d225ce 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,7 +154,8 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +RELEASE_DATE=$(date +%Y-%m-%d) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE}) ${COMMITS} " @@ -174,9 +175,17 @@ if [ -f "CHANGELOG.md" ]; then fi echo -e "${GREEN}โœ… CHANGELOG updated${NC}\n" +# Step 6b: Update the releases wall entry โ€” mechanical, derived from the +# CHANGELOG entry just composed. The fleet's HQ page reads releases/open-brainy.json +# directly; this used to be hand-written after every release (David: never +# again โ€” make it a step of the rail). +echo -e "${BLUE}5๏ธโƒฃโ–ธ Updating the releases wall...${NC}" +node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md +echo -e "${GREEN}โœ… Releases wall updated${NC}\n" + # Step 7: Create release commit echo -e "${BLUE}6๏ธโƒฃ Creating release commit...${NC}" -git add package.json package-lock.json CHANGELOG.md +git add package.json package-lock.json CHANGELOG.md releases/open-brainy.json git commit -m "chore(release): ${NEW_VERSION}" echo -e "${GREEN}โœ… Release commit created${NC}\n" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs new file mode 100644 index 00000000..998431da --- /dev/null +++ b/scripts/wall-entry.mjs @@ -0,0 +1,364 @@ +#!/usr/bin/env node +/** + * @module scripts/wall-entry + * @description The releases-wall entry, made mechanical. The fleet's HQ page + * reads one public JSON per product (releases/.json โ€” shape + * {product, entries:[{version, date, headline, items, url, thumb}], history}). + * Those entries were hand-written after every release; this script is the + * one door that composes one, so it never has to be typed by hand again. + * + * Two modes: + * + * 1. Generate + write in place (default): + * node wall-entry.mjs --product

--version --date \ + * --from-changelog [--file releases/

.json] + * Derives an entry from the CHANGELOG.md entry for (headline = the + * entry's first bullet, items = every bullet, trimmed of its trailing + * commit hash), prepends it to --file (default releases/.json, + * newest first), refusing by name if is already present, and + * validates the whole file's shape + ordering before and after writing. + * Both engines run this identically, each against its own repo's + * releases/.json โ€” the wall file always lives beside the + * CHANGELOG it is derived from, never in another repo. + * + * 2. Validate only (--check): + * node wall-entry.mjs --check --file + * Validates the file's exact key set (top-level and per-entry), field + * types, and strict-descending semver ordering with no duplicates. + * Read-only; never writes. Exit 0 = clean, exit 1 = named violations + * printed to stderr. + * + * No dependencies โ€” CHANGELOG parsing, semver comparison, and JSON shape + * checking are all hand-rolled below. + */ + +import { readFileSync, writeFileSync, existsSync } from 'node:fs' + +const ENTRY_KEYS = ['version', 'date', 'headline', 'items', 'url', 'thumb'] +const FILE_KEYS = ['product', 'entries', 'history'] + +// The public release-page URL pattern, by product โ€” only products with a +// PUBLIC forge repo get a derived link. A product without an entry here +// (e.g. "brainy", whose repo is private) gets url: null, matching every +// entry the fleet has shipped for it so far โ€” a private link would 404 for +// anyone reading the public HQ page. +const RELEASE_URL_PATTERNS = { + 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, +} + +/** + * Parse argv into a flag map. `--flag value` sets a string; `--flag` alone + * (end of argv, or followed by another `--flag`) sets boolean true. + * @param {string[]} argv + * @returns {Record} + */ +function parseArgs(argv) { + /** @type {Record} */ + const args = {} + for (let i = 0; i < argv.length; i++) { + const a = argv[i] + if (!a.startsWith('--')) continue + const key = a.slice(2) + const next = argv[i + 1] + if (next === undefined || next.startsWith('--')) { + args[key] = true + } else { + args[key] = next + i++ + } + } + return args +} + +/** + * Print a loud, named error and exit 1. Every refusal in this script goes + * through here so the failure mode is always the same shape: "wall-entry: ". + * @param {string} message + * @returns {never} + */ +function fail(message) { + console.error(`wall-entry: ${message}`) + process.exit(1) +} + +/** + * @param {string} version + * @returns {{major: number, minor: number, patch: number, pre: string | null} | null} + */ +function parseSemver(version) { + const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version) + if (!m) return null + return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null } +} + +/** + * @param {string} a + * @param {string} b + * @returns {number} positive if a > b, negative if a < b, 0 if equal. + */ +function compareSemver(a, b) { + const pa = parseSemver(a) + const pb = parseSemver(b) + if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`) + if (pa.major !== pb.major) return pa.major - pb.major + if (pa.minor !== pb.minor) return pa.minor - pb.minor + if (pa.patch !== pb.patch) return pa.patch - pb.patch + if (pa.pre === pb.pre) return 0 + if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version + if (pb.pre === null) return -1 + return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0 +} + +/** + * Validate a wall file's full shape: top-level keys, per-entry keys and + * field types, and strict-descending semver ordering with no duplicates. + * Collects every violation instead of failing on the first, so --check + * reports the whole picture in one pass. + * @param {unknown} data + * @returns {string[]} Violation messages; empty means the file is clean. + */ +function validateShape(data) { + /** @type {string[]} */ + const errors = [] + + if (typeof data !== 'object' || data === null || Array.isArray(data)) { + return ['top level: expected a JSON object'] + } + const obj = /** @type {Record} */ (data) + + const topKeys = Object.keys(obj) + const missingTop = FILE_KEYS.filter((k) => !(k in obj)) + const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k)) + if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`) + if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`) + + if (typeof obj.product !== 'string' || obj.product.trim() === '') { + errors.push('top level: "product" must be a non-empty string') + } + if (typeof obj.history !== 'string' || obj.history.trim() === '') { + errors.push('top level: "history" must be a non-empty string') + } + if (!Array.isArray(obj.entries)) { + errors.push('top level: "entries" must be an array') + return errors // nothing further to check without an array + } + + const entries = /** @type {unknown[]} */ (obj.entries) + entries.forEach((rawEntry, i) => { + const label = `entries[${i}]` + if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) { + errors.push(`${label}: expected an object`) + return + } + const entry = /** @type {Record} */ (rawEntry) + const keys = Object.keys(entry) + const missing = ENTRY_KEYS.filter((k) => !(k in entry)) + const extra = keys.filter((k) => !ENTRY_KEYS.includes(k)) + if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) + if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) + + if (typeof entry.version !== 'string' || !parseSemver(entry.version)) { + errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`) + } + if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) { + errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`) + } + if (typeof entry.headline !== 'string' || entry.headline.trim() === '') { + errors.push(`${label}: "headline" must be a non-empty string`) + } + if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { + errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) + } + if (!(entry.url === null || typeof entry.url === 'string')) { + errors.push(`${label}: "url" must be a string or null`) + } + if (!(entry.thumb === null || typeof entry.thumb === 'string')) { + errors.push(`${label}: "thumb" must be a string or null`) + } + }) + + // Ordering: newest first, strictly descending, no duplicate versions โ€” + // checked only over entries whose version parsed (a bad version is + // already reported above; comparing it too would just be noise). + const versioned = entries + .map((e, i) => ({ i, version: /** @type {any} */ (e)?.version })) + .filter((e) => typeof e.version === 'string' && parseSemver(e.version)) + for (let i = 0; i < versioned.length - 1; i++) { + const a = versioned[i] + const b = versioned[i + 1] + const cmp = compareSemver(a.version, b.version) + if (cmp === 0) { + errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`) + } else if (cmp < 0) { + errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) โ€” not newest-first`) + } + } + + return errors +} + +/** + * Extract one version's entry body from a standard-version-style CHANGELOG.md + * (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines + * until the next heading or EOF). + * @param {string} changelog + * @param {string} version + * @returns {string[]} Bullet lines, trimmed of their leading "- " and + * trailing " (hash)". + */ +function extractChangelogBullets(changelog, version) { + const lines = changelog.split('\n') + const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/ + let start = -1 + for (let i = 0; i < lines.length; i++) { + const m = headingRe.exec(lines[i]) + if (m && m[1] === version) { + start = i + 1 + break + } + } + if (start === -1) { + fail( + `version ${version} has no CHANGELOG entry yet โ€” run this after the CHANGELOG step composes "### [${version}]", not before`, + ) + } + /** @type {string[]} */ + const bullets = [] + for (let i = start; i < lines.length; i++) { + if (headingRe.test(lines[i])) break // next entry starts + const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim()) + if (lines[i].trim().startsWith('- ') && bulletMatch) { + const text = bulletMatch[1].trim() + if (text) bullets.push(text) + } + } + if (bullets.length === 0) { + fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`) + } + return bullets +} + +/** + * Derive a wall entry from a CHANGELOG.md. + * @param {{product: string, version: string, date: string, changelogPath: string, url?: string | null, thumb?: string | null}} opts + * @returns {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} + */ +function deriveEntry({ product, version, date, changelogPath, url, thumb }) { + if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) + if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) { + fail(`--date "${date}" is not a YYYY-MM-DD date`) + } + if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`) + + const changelog = readFileSync(changelogPath, 'utf8') + const items = extractChangelogBullets(changelog, version) + const headline = items[0] + + const resolvedUrl = url !== undefined ? url : (RELEASE_URL_PATTERNS[product]?.(version) ?? null) + const resolvedThumb = thumb !== undefined ? thumb : null + + return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } +} + +/** + * Load and shape-validate a wall file. + * @param {string} filePath + * @returns {Record} + */ +function loadWallFile(filePath) { + if (!existsSync(filePath)) fail(`--file "${filePath}" does not exist`) + /** @type {unknown} */ + let data + try { + data = JSON.parse(readFileSync(filePath, 'utf8')) + } catch (err) { + fail(`--file "${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) + } + const errors = validateShape(data) + if (errors.length) { + fail(`--file "${filePath}" fails shape validation before any write โ€”\n ${errors.join('\n ')}`) + } + return /** @type {Record} */ (data) +} + +/** + * Prepend `entry` to the wall file at `filePath`, refusing by name if the + * version is already present, validating before and after, and writing the + * file back with the repo's exact formatting (2-space JSON, trailing newline). + * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {string} filePath + * @param {string | undefined} expectedProduct + */ +function applyEntry(entry, filePath, expectedProduct) { + const wall = loadWallFile(filePath) + + if (expectedProduct && wall.product !== expectedProduct) { + fail( + `--file "${filePath}" has product "${wall.product}", but --product "${expectedProduct}" was given โ€” refusing a cross-product write`, + ) + } + + if (wall.entries.some((e) => e.version === entry.version)) { + fail(`refusing โ€” version ${entry.version} is already present in "${filePath}"`) + } + + wall.entries = [entry, ...wall.entries] + + const postErrors = validateShape(wall) + if (postErrors.length) { + fail(`the entry for ${entry.version} would leave "${filePath}" invalid โ€”\n ${postErrors.join('\n ')}`) + } + + writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') + console.log(`wall-entry: wrote v${entry.version} to "${filePath}" (${wall.entries.length} entries, newest first)`) +} + +function main() { + const args = parseArgs(process.argv.slice(2)) + + if (args.check) { + const filePath = /** @type {string | undefined} */ (args.file) ?? + (typeof args.product === 'string' ? `releases/${args.product}.json` : undefined) + if (!filePath) fail('--check needs --file (or --product to default to releases/.json)') + const wall = loadWallFile(/** @type {string} */ (filePath)) + console.log(`wall-entry --check: "${filePath}" OK โ€” product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) + process.exit(0) + } + + // Generate mode (default): --product, --version, --date, --from-changelog required. + const product = /** @type {string | undefined} */ (args.product) + const version = /** @type {string | undefined} */ (args.version) + const date = /** @type {string | undefined} */ (args.date) + const fromChangelog = /** @type {string | undefined} */ (args['from-changelog']) + + const missing = [] + if (!product) missing.push('--product') + if (!version) missing.push('--version') + if (!date) missing.push('--date') + if (!fromChangelog) missing.push('--from-changelog') + if (missing.length) { + fail( + `missing required flag(s): ${missing.join(', ')}\n` + + 'Usage:\n' + + ' wall-entry.mjs --product

--version --date --from-changelog [--file releases/

.json]\n' + + ' wall-entry.mjs --check --file ', + ) + } + + const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url) + const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb) + + const entry = deriveEntry({ + product: /** @type {string} */ (product), + version: /** @type {string} */ (version), + date: /** @type {string} */ (date), + changelogPath: /** @type {string} */ (fromChangelog), + url: urlArg, + thumb: thumbArg, + }) + + const filePath = /** @type {string} */ (args.file ?? `releases/${product}.json`) + applyEntry(entry, filePath, /** @type {string} */ (product)) +} + +main() diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts new file mode 100644 index 00000000..fc41731c --- /dev/null +++ b/tests/unit/release/wall-entry.test.ts @@ -0,0 +1,216 @@ +/** + * scripts/wall-entry.mjs โ€” the mechanical releases-wall entry. + * + * The script's only real interface is its CLI (it has no importable + * exports by design โ€” one door, no parallel API to drift from it), so + * these tests spawn it exactly as scripts/release.sh does: as a child + * process, against a temp copy of a wall file and a fixture CHANGELOG, + * never against the repo's real releases/*.json. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { execFileSync } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const SCRIPT = join(process.cwd(), 'scripts/wall-entry.mjs') + +/** Run the script and capture the outcome without throwing on a non-zero exit. */ +function run(args: string[], cwd: string): { status: number; stdout: string; stderr: string } { + try { + const stdout = execFileSync('node', [SCRIPT, ...args], { cwd, encoding: 'utf8' }) + return { status: 0, stdout, stderr: '' } + } catch (err: any) { + return { status: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' } + } +} + +const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' + +/** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ +function buildChangelog(entries: Array<{ version: string; date: string; bullets: string[] }>): string { + const body = entries + .map( + (e) => + `### [${e.version}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/vX...v${e.version}) (${e.date})\n\n` + + e.bullets.map((b) => `- ${b} (abc1234)`).join('\n') + + '\n', + ) + .join('\n') + return CHANGELOG_HEADER + '\n' + body +} + +function wallFile(product: string, entries: unknown[]): string { + return JSON.stringify( + { product, entries, history: 'Earlier releases are recorded in CHANGELOG.md in this repository.' }, + null, + 2, + ) + '\n' +} + +const BASE_ENTRY = { + version: '10.4.11', + date: '2026-09-02', + headline: 'A faster open', + items: ['A faster open.'], + url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11', + thumb: null, +} + +let dir: string + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) +}) + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }) +}) + +describe('wall-entry.mjs โ€” generate + prepend', () => { + it('derives headline from the first bullet and items from every bullet, hashes stripped', () => { + writeFileSync( + join(dir, 'CHANGELOG.md'), + buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), + ) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + expect(result.status).toBe(0) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries).toHaveLength(2) + expect(wall.entries[0]).toEqual({ + version: '10.4.12', + date: '2026-09-03', + headline: 'fix(wall): mechanize the entry', + items: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'], + url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.12', + thumb: null, + }) + // the older entry stays put, still second + expect(wall.entries[1].version).toBe('10.4.11') + }) + + it('prepends newest-first โ€” the new entry lands at index 0 ahead of every existing one', () => { + writeFileSync( + join(dir, 'CHANGELOG.md'), + buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }]), + ) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + + run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) + }) + + it('derives no URL (null) for a product with no known public release-page pattern', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }])) + + run(['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries[0].url).toBeNull() + expect(wall.entries[0].thumb).toBeNull() + }) + + it('refuses by name when the version is already present, and leaves the file untouched', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) + const before = wallFile('open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'wall.json'), before) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/refusing.*10\.4\.11.*already present/i) + expect(readFileSync(join(dir, 'wall.json'), 'utf8')).toBe(before) // untouched + }) + + it('refuses when the CHANGELOG has no entry yet for the target version', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [])) + + const result = run( + ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) + }) + + it('refuses a cross-product write when --product does not match the target file', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + + const result = run( + ['--product', 'brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/product "open-brainy".*--product "brainy"/i) + }) +}) + +describe('wall-entry.mjs โ€” --check', () => { + it('passes a well-formed, newest-first file with no duplicates', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/OK/) + }) + + it('catches a missing entry key', () => { + const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'], url: null } // no "thumb" + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/missing key\(s\) thumb/) + }) + + it('catches an unexpected top-level key', () => { + const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) + raw.extra = 'not allowed' + writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/unexpected key\(s\) extra/) + }) + + it('catches entries that are not newest-first', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, version: '10.4.10' }, BASE_ENTRY])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/not newest-first/) + }) + + it('catches a duplicate version even with identical entries', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/duplicate version 10\.4\.11/) + }) + + it('catches an empty items array', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, items: [] }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/"items" must be a non-empty array/) + }) + + it('catches a malformed date', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, date: '09/03/2026' }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/"date" must be a YYYY-MM-DD string/) + }) +}) From adcb883e67ab82b749d37a510ab66323ae1da64e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:51:33 -0700 Subject: [PATCH 22/24] ci(release): publish the wall entry to the shared releases repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rail used to write releases/open-brainy.json (and, before that, also carried the product engine's releases/brainy.json) in this repo. It now clones (or refreshes a cached clone of) soulcraftlabs/releases on The Source, prepends the derived entry to open-brainy.json there (replacing any entry for the same version so a re-run is idempotent), and pushes main directly. Any failure โ€” clone, shape validation, commit, or a rejected push โ€” exits non-zero naming the cure; nothing is ever skipped. Both wall files are gone from this repo โ€” the shared repo is the one home HQ reads. --dry-run derives and prints the entry without touching any clone or remote. Tests point --remote/--cache-dir at a throwaway local bare repo and cache dir, never the real ones. --- releases/brainy.json | 76 -------- releases/open-brainy.json | 136 ------------- scripts/release.sh | 13 +- scripts/wall-entry.mjs | 253 ++++++++++++++++++------ tests/unit/release/wall-entry.test.ts | 271 +++++++++++++++++++++----- 5 files changed, 423 insertions(+), 326 deletions(-) delete mode 100644 releases/brainy.json delete mode 100644 releases/open-brainy.json diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door โ€” correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open โ€” measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild โ€” measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step โ€” the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3โ€“4ร— faster imports", - "items": [ - "Every runtime embeds with the same full-precision model โ€” search quality no longer depends on where you run.", - "Bulk embedding measured 3.1โ€“4.2ร— faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine โ€” Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place โ€” no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 9f1cd239..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.11", - "date": "2026-09-02", - "headline": "Hybrid finds filter before they hydrate, one owner per shutdown, and a faster open", - "items": [ - "Hybrid finds (query/vector combined with a filter, including connected and fusion finds) now filter first and hydrate only the page โ€” one batchGet of exactly the requested rows, instead of hydrating everything the search side found. Fixes a bug where any page after the first came back empty.", - "A brain now has exactly one shutdown owner โ€” a host and its engine no longer race to close the same store, and a follow-up flush requested during a running flush is handed off cleanly instead of ever risking a stall.", - "find({ path }) and other path-scoped VFS searches now serve a real range over the indexed path (O(log n)) instead of refusing the query outright โ€” both scoped and recursive:false searches were silently broken before this.", - "Open no longer rescans a brain's whole fact log on every open โ€” sealed segments the manifest already accounts for are skipped, collapsing a multi-second open term to near-zero on large brains.", - "commitTransaction() now refuses by name if single-ops are still pending, and a read-only open no longer writes clean-shutdown evidence it didn't earn โ€” two correctness invariants that were previously assumed, not enforced." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11", - "thumb": null - }, - { - "version": "10.4.10", - "date": "2026-09-02", - "headline": "A planner door for indexes, batched containment repair, and a fixed near()", - "items": [ - "An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.", - "repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.", - "find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10", - "thumb": null - }, - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows โ€” correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind โ€” four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open โ€” measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass โ€” parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index โ€” operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure โ€” never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy โ€” the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints โ€” a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} diff --git a/scripts/release.sh b/scripts/release.sh index 07d225ce..142fa06f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -176,16 +176,21 @@ fi echo -e "${GREEN}โœ… CHANGELOG updated${NC}\n" # Step 6b: Update the releases wall entry โ€” mechanical, derived from the -# CHANGELOG entry just composed. The fleet's HQ page reads releases/open-brainy.json -# directly; this used to be hand-written after every release (David: never -# again โ€” make it a step of the rail). +# CHANGELOG entry just composed. The fleet's HQ page reads open-brainy.json +# from the one shared releases repo, soulcraftlabs/releases on The Source โ€” +# this used to be hand-written after every release (David: never again โ€” +# make it a step of the rail, landed in the one shared home; this repo no +# longer hosts its own copy). This step clones/fetches that repo into a +# local cache, prepends the entry, and pushes it directly โ€” a real +# cross-repo push, refusing loudly (never skipping) on any +# clone/validation/commit/push failure. echo -e "${BLUE}5๏ธโƒฃโ–ธ Updating the releases wall...${NC}" node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md echo -e "${GREEN}โœ… Releases wall updated${NC}\n" # Step 7: Create release commit echo -e "${BLUE}6๏ธโƒฃ Creating release commit...${NC}" -git add package.json package-lock.json CHANGELOG.md releases/open-brainy.json +git add package.json package-lock.json CHANGELOG.md git commit -m "chore(release): ${NEW_VERSION}" echo -e "${GREEN}โœ… Release commit created${NC}\n" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs index 998431da..043341eb 100644 --- a/scripts/wall-entry.mjs +++ b/scripts/wall-entry.mjs @@ -2,40 +2,76 @@ /** * @module scripts/wall-entry * @description The releases-wall entry, made mechanical. The fleet's HQ page - * reads one public JSON per product (releases/.json โ€” shape - * {product, entries:[{version, date, headline, items, url, thumb}], history}). - * Those entries were hand-written after every release; this script is the - * one door that composes one, so it never has to be typed by hand again. + * reads one public JSON per product from the ONE releases repo on The Source + * (soulcraftlabs/releases, files .json at its root โ€” shape + * {product, entries:[{version, date, headline, items, url, thumb?}]}), at + * https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/.json. + * Those entries were hand-written after every release, then briefly written + * into this repo's own releases/.json; this script is the one door + * that composes an entry and lands it in the shared repo, so it is never + * hand-written and never forked across repos again. * * Two modes: * - * 1. Generate + write in place (default): + * 1. Generate + publish (default): * node wall-entry.mjs --product

--version --date \ - * --from-changelog [--file releases/

.json] + * --from-changelog * Derives an entry from the CHANGELOG.md entry for (headline = the * entry's first bullet, items = every bullet, trimmed of its trailing - * commit hash), prepends it to --file (default releases/.json, - * newest first), refusing by name if is already present, and - * validates the whole file's shape + ordering before and after writing. - * Both engines run this identically, each against its own repo's - * releases/.json โ€” the wall file always lives beside the - * CHANGELOG it is derived from, never in another repo. + * commit hash), then: + * - clones (or, if a cached clone already exists, fetches and resets) + * the releases repo into a local cache directory, + * - prepends the entry to /

.json, newest first โ€” replacing + * any existing entry for the same version so a re-run is idempotent, + * - validates the file's shape before and after, + * - commits the change as "chore(wall):

" and pushes main. + * A failure at any step (clone, validation, commit, push, a + * non-fast-forward remote) exits non-zero naming the cure. Nothing is + * ever skipped โ€” the wall either lands correctly or the release fails. * - * 2. Validate only (--check): - * node wall-entry.mjs --check --file - * Validates the file's exact key set (top-level and per-entry), field - * types, and strict-descending semver ordering with no duplicates. - * Read-only; never writes. Exit 0 = clean, exit 1 = named violations - * printed to stderr. + * 2. Dry run: + * node wall-entry.mjs --dry-run --product

--version \ + * --date --from-changelog + * Derives the entry exactly as above and prints it, along with the file + * it would be written to, but touches no clone and no remote โ€” usable + * from a fresh checkout with no cache and no network. * - * No dependencies โ€” CHANGELOG parsing, semver comparison, and JSON shape - * checking are all hand-rolled below. + * 3. Validate only (--check): + * node wall-entry.mjs --check --file + * Validates an arbitrary wall file's exact key set (top-level and + * per-entry), field types, and strict-descending semver ordering with + * no duplicates. Read-only; never writes. Exit 0 = clean, exit 1 = + * named violations printed to stderr. + * + * The remote and the local cache directory are each overridable + * (--remote / --cache-dir, or WALL_ENTRY_RELEASES_REMOTE / + * WALL_ENTRY_RELEASES_CACHE_DIR) so tests can point at a throwaway local + * bare repo and a throwaway cache directory โ€” never the real remote or the + * real developer cache. + * + * No dependencies beyond the system `git` binary โ€” CHANGELOG parsing, + * semver comparison, and JSON shape checking are all hand-rolled below. */ -import { readFileSync, writeFileSync, existsSync } from 'node:fs' +import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs' +import { execFileSync } from 'node:child_process' +import { homedir } from 'node:os' +import { dirname, join } from 'node:path' -const ENTRY_KEYS = ['version', 'date', 'headline', 'items', 'url', 'thumb'] -const FILE_KEYS = ['product', 'entries', 'history'] +const DEFAULT_REMOTE = 'git@source.soulcraft.com:soulcraftlabs/releases.git' + +/** @returns {string} */ +function defaultCacheDir() { + const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache') + return join(base, 'soulcraft-releases') +} + +// Required on every entry; "thumb" is optional (may be absent, or present as +// string | null) โ€” matching the HQ contract's {..., thumb?}. +const ENTRY_REQUIRED_KEYS = ['version', 'date', 'headline', 'items', 'url'] +const ENTRY_OPTIONAL_KEYS = ['thumb'] +const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS] +const FILE_KEYS = ['product', 'entries'] // The public release-page URL pattern, by product โ€” only products with a // PUBLIC forge repo get a derived link. A product without an entry here @@ -110,10 +146,11 @@ function compareSemver(a, b) { } /** - * Validate a wall file's full shape: top-level keys, per-entry keys and - * field types, and strict-descending semver ordering with no duplicates. - * Collects every violation instead of failing on the first, so --check - * reports the whole picture in one pass. + * Validate a wall file's full shape: top-level keys ("product", "entries" โ€” + * no more, no less), per-entry keys and field types ("thumb" optional), and + * strict-descending semver ordering with no duplicates. Collects every + * violation instead of failing on the first, so a caller reports the whole + * picture in one pass. * @param {unknown} data * @returns {string[]} Violation messages; empty means the file is clean. */ @@ -135,9 +172,6 @@ function validateShape(data) { if (typeof obj.product !== 'string' || obj.product.trim() === '') { errors.push('top level: "product" must be a non-empty string') } - if (typeof obj.history !== 'string' || obj.history.trim() === '') { - errors.push('top level: "history" must be a non-empty string') - } if (!Array.isArray(obj.entries)) { errors.push('top level: "entries" must be an array') return errors // nothing further to check without an array @@ -152,8 +186,8 @@ function validateShape(data) { } const entry = /** @type {Record} */ (rawEntry) const keys = Object.keys(entry) - const missing = ENTRY_KEYS.filter((k) => !(k in entry)) - const extra = keys.filter((k) => !ENTRY_KEYS.includes(k)) + const missing = ENTRY_REQUIRED_KEYS.filter((k) => !(k in entry)) + const extra = keys.filter((k) => !ENTRY_ALLOWED_KEYS.includes(k)) if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) @@ -172,8 +206,8 @@ function validateShape(data) { if (!(entry.url === null || typeof entry.url === 'string')) { errors.push(`${label}: "url" must be a string or null`) } - if (!(entry.thumb === null || typeof entry.thumb === 'string')) { - errors.push(`${label}: "thumb" must be a string or null`) + if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) { + errors.push(`${label}: "thumb" must be a string or null when present`) } }) @@ -266,43 +300,110 @@ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { * @returns {Record} */ function loadWallFile(filePath) { - if (!existsSync(filePath)) fail(`--file "${filePath}" does not exist`) + if (!existsSync(filePath)) fail(`"${filePath}" does not exist`) /** @type {unknown} */ let data try { data = JSON.parse(readFileSync(filePath, 'utf8')) } catch (err) { - fail(`--file "${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) + fail(`"${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) } const errors = validateShape(data) if (errors.length) { - fail(`--file "${filePath}" fails shape validation before any write โ€”\n ${errors.join('\n ')}`) + fail(`"${filePath}" fails shape validation โ€”\n ${errors.join('\n ')}`) } return /** @type {Record} */ (data) } /** - * Prepend `entry` to the wall file at `filePath`, refusing by name if the - * version is already present, validating before and after, and writing the - * file back with the repo's exact formatting (2-space JSON, trailing newline). - * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry - * @param {string} filePath - * @param {string | undefined} expectedProduct + * Run a git command, throwing an Error whose message is git's own stderr + * (trimmed) on failure โ€” every caller wraps this to name the cure. + * @param {string[]} args + * @param {string} cwd + * @returns {string} stdout, trimmed. */ -function applyEntry(entry, filePath, expectedProduct) { - const wall = loadWallFile(filePath) +function git(args, cwd) { + try { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim() + } catch (err) { + const stderr = /** @type {any} */ (err).stderr + const message = (typeof stderr === 'string' && stderr.trim()) || /** @type {Error} */ (err).message + throw new Error(message) + } +} - if (expectedProduct && wall.product !== expectedProduct) { +/** + * Ensure a clean, up-to-date local clone of the releases repo at + * `cacheDir`, checked out on `main` โ€” cloning fresh if `cacheDir` has no + * `.git`, otherwise fetching and hard-resetting onto `origin/main` (so a + * stray local commit or edit left by a previous failed run can never leak + * into the next one). + * @param {string} remote + * @param {string} cacheDir + */ +function ensureReleasesClone(remote, cacheDir) { + if (existsSync(join(cacheDir, '.git'))) { + try { + git(['remote', 'set-url', 'origin', remote], cacheDir) + git(['fetch', '--prune', 'origin'], cacheDir) + git(['checkout', 'main'], cacheDir) + git(['reset', '--hard', 'origin/main'], cacheDir) + git(['clean', '-fd'], cacheDir) + } catch (err) { + fail( + `cannot refresh the cached releases checkout at "${cacheDir}" from "${remote}" โ€” ${/** @type {Error} */ (err).message}\n` + + ` cure: delete "${cacheDir}" and re-run so it re-clones from scratch, or confirm SSH access with "ssh -T git@source.soulcraft.com"`, + ) + } + return + } + + mkdirSync(dirname(cacheDir), { recursive: true }) + try { + git(['clone', remote, cacheDir], dirname(cacheDir)) + } catch (err) { fail( - `--file "${filePath}" has product "${wall.product}", but --product "${expectedProduct}" was given โ€” refusing a cross-product write`, + `cannot clone "${remote}" โ€” ${/** @type {Error} */ (err).message}\n` + + ` cure: confirm SSH access with "ssh -T git@source.soulcraft.com" and that the soulcraftlabs/releases repo exists yet`, ) } + try { + git(['checkout', 'main'], cacheDir) + } catch (err) { + fail( + `cloned "${remote}" into "${cacheDir}" but could not check out "main" โ€” ${/** @type {Error} */ (err).message}\n` + + ` cure: confirm the releases repo's default branch is named "main"`, + ) + } +} - if (wall.entries.some((e) => e.version === entry.version)) { - fail(`refusing โ€” version ${entry.version} is already present in "${filePath}"`) +/** + * Prepend `entry` to the wall at `/.json`, replacing any + * existing entry for the same version (idempotent re-runs), validating + * before and after, committing, and pushing โ€” or refusing loudly, naming + * the cure, at whichever step fails. + * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {string} product + * @param {string} remote + * @param {string} cacheDir + */ +function publishEntry(entry, product, remote, cacheDir) { + ensureReleasesClone(remote, cacheDir) + + const filePath = join(cacheDir, `${product}.json`) + if (!existsSync(filePath)) { + fail( + `"${filePath}" does not exist in the releases repo โ€” cure: seed "${product}.json" at the repo root first (it must exist before any release rail can prepend to it)`, + ) + } + const wall = loadWallFile(filePath) + + if (wall.product !== product) { + fail(`"${filePath}" has product "${wall.product}", but --product "${product}" was given โ€” refusing a cross-product write`) } - wall.entries = [entry, ...wall.entries] + const replacing = wall.entries.some((e) => e.version === entry.version) + wall.entries = [entry, ...wall.entries.filter((e) => e.version !== entry.version)] const postErrors = validateShape(wall) if (postErrors.length) { @@ -310,22 +411,48 @@ function applyEntry(entry, filePath, expectedProduct) { } writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') - console.log(`wall-entry: wrote v${entry.version} to "${filePath}" (${wall.entries.length} entries, newest first)`) + + const status = git(['status', '--porcelain', '--', `${product}.json`], cacheDir) + if (status === '') { + console.log(`wall-entry: "${product}.json" already carries an identical entry for ${entry.version} โ€” nothing to commit or push`) + return + } + + try { + git(['add', `${product}.json`], cacheDir) + git(['commit', '-m', `chore(wall): ${product} ${entry.version}`], cacheDir) + } catch (err) { + fail(`cannot commit the wall entry in "${cacheDir}" โ€” ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`) + } + + try { + git(['push', 'origin', 'main'], cacheDir) + } catch (err) { + fail( + `push to "${remote}" failed (likely a non-fast-forward โ€” another release landed on main first) โ€” ${/** @type {Error} */ (err).message}\n` + + ` cure: re-run this release step; it re-fetches and resets onto the latest origin/main before retrying`, + ) + } + + const sha = git(['rev-parse', 'HEAD'], cacheDir) + console.log( + `wall-entry: ${replacing ? 'replaced' : 'wrote'} v${entry.version} in "${product}.json" (${wall.entries.length} entries, newest first) โ€” pushed ${sha} to ${remote} main`, + ) } function main() { const args = parseArgs(process.argv.slice(2)) if (args.check) { - const filePath = /** @type {string | undefined} */ (args.file) ?? - (typeof args.product === 'string' ? `releases/${args.product}.json` : undefined) - if (!filePath) fail('--check needs --file (or --product to default to releases/.json)') + const filePath = /** @type {string | undefined} */ (args.file) + if (!filePath) fail('--check needs --file ') const wall = loadWallFile(/** @type {string} */ (filePath)) console.log(`wall-entry --check: "${filePath}" OK โ€” product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) process.exit(0) } - // Generate mode (default): --product, --version, --date, --from-changelog required. + // Generate mode (default, also covers --dry-run): --product, --version, + // --date, --from-changelog required. const product = /** @type {string | undefined} */ (args.product) const version = /** @type {string | undefined} */ (args.version) const date = /** @type {string | undefined} */ (args.date) @@ -340,8 +467,8 @@ function main() { fail( `missing required flag(s): ${missing.join(', ')}\n` + 'Usage:\n' + - ' wall-entry.mjs --product

--version --date --from-changelog [--file releases/

.json]\n' + - ' wall-entry.mjs --check --file ', + ' wall-entry.mjs --product

--version --date --from-changelog [--dry-run]\n' + + ' wall-entry.mjs --check --file ', ) } @@ -357,8 +484,16 @@ function main() { thumb: thumbArg, }) - const filePath = /** @type {string} */ (args.file ?? `releases/${product}.json`) - applyEntry(entry, filePath, /** @type {string} */ (product)) + const remote = /** @type {string} */ (args.remote ?? process.env.WALL_ENTRY_RELEASES_REMOTE ?? DEFAULT_REMOTE) + const cacheDir = /** @type {string} */ (args['cache-dir'] ?? process.env.WALL_ENTRY_RELEASES_CACHE_DIR ?? defaultCacheDir()) + + if (args['dry-run']) { + console.log(`wall-entry --dry-run: would write to "${join(cacheDir, `${product}.json`)}" in ${remote} (main), pushed as "chore(wall): ${product} ${version}"`) + console.log(JSON.stringify(entry, null, 2)) + process.exit(0) + } + + publishEntry(entry, /** @type {string} */ (product), remote, cacheDir) } main() diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts index fc41731c..7f96da25 100644 --- a/tests/unit/release/wall-entry.test.ts +++ b/tests/unit/release/wall-entry.test.ts @@ -4,12 +4,15 @@ * The script's only real interface is its CLI (it has no importable * exports by design โ€” one door, no parallel API to drift from it), so * these tests spawn it exactly as scripts/release.sh does: as a child - * process, against a temp copy of a wall file and a fixture CHANGELOG, - * never against the repo's real releases/*.json. + * process, against a fixture CHANGELOG and a throwaway local bare repo + * standing in for git@source.soulcraft.com:soulcraftlabs/releases.git + * (--remote) plus a throwaway cache directory (--cache-dir) standing in + * for ~/.cache/soulcraft-releases โ€” never the real remote, never the + * real developer cache. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { execFileSync } from 'node:child_process' -import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs' +import { mkdtempSync, rmSync, writeFileSync, readFileSync, chmodSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -25,6 +28,10 @@ function run(args: string[], cwd: string): { status: number; stdout: string; std } } +function git(args: string[], cwd: string): string { + return execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8' }).trim() +} + const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' /** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ @@ -41,11 +48,7 @@ function buildChangelog(entries: Array<{ version: string; date: string; bullets: } function wallFile(product: string, entries: unknown[]): string { - return JSON.stringify( - { product, entries, history: 'Earlier releases are recorded in CHANGELOG.md in this repository.' }, - null, - 2, - ) + '\n' + return JSON.stringify({ product, entries }, null, 2) + '\n' } const BASE_ENTRY = { @@ -57,31 +60,76 @@ const BASE_ENTRY = { thumb: null, } +/** A throwaway bare repo standing in for the real soulcraftlabs/releases remote. */ +function initBareRemote(): string { + const remoteDir = mkdtempSync(join(tmpdir(), 'wall-remote-')) + execFileSync('git', ['init', '--bare', '-b', 'main', remoteDir]) + return remoteDir +} + +/** Seed the bare remote with an initial .json, via a throwaway clone. */ +function seedRemote(remoteDir: string, product: string, entries: unknown[]): void { + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + writeFileSync(join(seedDir, `${product}.json`), wallFile(product, entries)) + git(['add', `${product}.json`], seedDir) + git(['commit', '-m', 'seed'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) +} + +/** Read .json back out of the bare remote's main tip, via a throwaway clone. */ +function readRemote(remoteDir: string, product: string): any { + const readDir = mkdtempSync(join(tmpdir(), 'wall-read-')) + execFileSync('git', ['clone', remoteDir, readDir], { stdio: 'ignore' }) + const data = JSON.parse(readFileSync(join(readDir, `${product}.json`), 'utf8')) + rmSync(readDir, { recursive: true, force: true }) + return data +} + +/** Reject every push โ€” stands in for any push failure (including a genuine + * non-fast-forward raced by a concurrent release rail), which this script + * treats identically: refuse loudly, name the cure, touch nothing further. */ +function makeRemoteRejectPushes(remoteDir: string): void { + const hookPath = join(remoteDir, 'hooks', 'pre-receive') + writeFileSync(hookPath, '#!/bin/sh\necho "remote: simulated push rejection" >&2\nexit 1\n') + chmodSync(hookPath, 0o755) +} + let dir: string +let remoteDir: string +let cacheDir: string beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) + remoteDir = initBareRemote() + cacheDir = join(mkdtempSync(join(tmpdir(), 'wall-cache-')), 'soulcraft-releases') }) afterEach(() => { rmSync(dir, { recursive: true, force: true }) + rmSync(remoteDir, { recursive: true, force: true }) + rmSync(cacheDir, { recursive: true, force: true }) }) -describe('wall-entry.mjs โ€” generate + prepend', () => { - it('derives headline from the first bullet and items from every bullet, hashes stripped', () => { +describe('wall-entry.mjs โ€” generate + publish', () => { + it('derives headline from the first bullet and items from every bullet, hashes stripped, and pushes it to the remote', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) writeFileSync( join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), ) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(0) + expect(result.stdout).toMatch(/wrote v10\.4\.12.*pushed/i) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'open-brainy') expect(wall.entries).toHaveLength(2) expect(wall.entries[0]).toEqual({ version: '10.4.12', @@ -96,68 +144,182 @@ describe('wall-entry.mjs โ€” generate + prepend', () => { }) it('prepends newest-first โ€” the new entry lands at index 0 ahead of every existing one', () => { - writeFileSync( - join(dir, 'CHANGELOG.md'), - buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }]), - ) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }])) - run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'open-brainy') expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) }) + it('replaces an entry with the same version instead of duplicating it โ€” idempotent re-runs', () => { + seedRemote(remoteDir, 'open-brainy', [ + { ...BASE_ENTRY, headline: 'stale headline, pre-fix' }, + { ...BASE_ENTRY, version: '10.4.10' }, + ]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: the corrected headline'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/replaced v10\.4\.11/i) + + const wall = readRemote(remoteDir, 'open-brainy') + expect(wall.entries).toHaveLength(2) // not 3 โ€” replaced, not duplicated + expect(wall.entries[0].version).toBe('10.4.11') + expect(wall.entries[0].headline).toBe('fix: the corrected headline') + expect(wall.entries[1].version).toBe('10.4.10') + }) + + it('a re-run with byte-identical content commits nothing and still succeeds', () => { + // headline always equals items[0] for a derived entry, so this fixture + // (unlike BASE_ENTRY, whose headline/items intentionally diverge for the + // shape-only tests below) has to keep the two in lockstep to ever roundtrip. + const stableEntry = { ...BASE_ENTRY, headline: 'A faster open.', items: ['A faster open.'] } + seedRemote(remoteDir, 'open-brainy', [stableEntry]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['A faster open.'] }])) + const before = readRemote(remoteDir, 'open-brainy') + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/nothing to commit/i) + expect(readRemote(remoteDir, 'open-brainy')).toEqual(before) + }) + it('derives no URL (null) for a product with no known public release-page pattern', () => { + seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }]) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }])) - run(['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + const result = run( + ['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'brainy') expect(wall.entries[0].url).toBeNull() expect(wall.entries[0].thumb).toBeNull() }) - it('refuses by name when the version is already present, and leaves the file untouched', () => { + it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => { + seedRemote(remoteDir, 'open-brainy', []) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) - const before = wallFile('open-brainy', [BASE_ENTRY]) - writeFileSync(join(dir, 'wall.json'), before) + const beforeSha = git(['rev-parse', 'main'], remoteDir) const result = run( - ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/refusing.*10\.4\.11.*already present/i) - expect(readFileSync(join(dir, 'wall.json'), 'utf8')).toBe(before) // untouched - }) - - it('refuses when the CHANGELOG has no entry yet for the target version', () => { - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [])) - - const result = run( - ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(1) expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) }) - it('refuses a cross-product write when --product does not match the target file', () => { - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + it('refuses by naming the cure when the remote cannot be cloned', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + const noSuchRemote = join(tmpdir(), 'wall-remote-does-not-exist-' + Date.now()) const result = run( - ['--product', 'brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', noSuchRemote, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/product "open-brainy".*--product "brainy"/i) + expect(result.stderr).toMatch(/cannot clone/i) + expect(result.stderr).toMatch(/cure:/i) + }) + + it('refuses by naming the cure, and touches no remote, when the fetched wall fails shape validation', () => { + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-broken-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + writeFileSync( + join(seedDir, 'open-brainy.json'), + JSON.stringify({ product: 'open-brainy', entries: [{ version: '10.4.11', date: '2026-09-02', items: ['x'], url: null }] }, null, 2), + ) + git(['add', 'open-brainy.json'], seedDir) + git(['commit', '-m', 'seed broken'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) + const beforeSha = git(['rev-parse', 'main'], remoteDir) + + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/fails shape validation/i) + expect(result.stderr).toMatch(/missing key\(s\) headline/i) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) + }) + + it('refuses by naming the cure when the remote rejects the push (stands in for a raced non-fast-forward)', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + makeRemoteRejectPushes(remoteDir) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/push to .* failed/i) + expect(result.stderr).toMatch(/cure:/i) + }) + + it('refuses a cross-product write when the file\'s "product" field does not match --product', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-mismatch-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + const corrupted = JSON.parse(readFileSync(join(seedDir, 'open-brainy.json'), 'utf8')) + corrupted.product = 'brainy' + writeFileSync(join(seedDir, 'open-brainy.json'), JSON.stringify(corrupted, null, 2) + '\n') + git(['add', 'open-brainy.json'], seedDir) + git(['commit', '-m', 'corrupt product field'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) + + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/product "brainy".*--product "open-brainy"/i) + }) +}) + +describe('wall-entry.mjs โ€” --dry-run', () => { + it('prints the entry and the target path, and touches neither the cache dir nor the remote', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: a dry run'] }])) + const beforeSha = git(['rev-parse', 'main'], remoteDir) + + const result = run( + ['--dry-run', '--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/would write to/i) + expect(result.stdout).toMatch(/"version": "10\.4\.12"/) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) }) }) @@ -169,21 +331,28 @@ describe('wall-entry.mjs โ€” --check', () => { expect(result.stdout).toMatch(/OK/) }) + it('passes a file where "thumb" is entirely absent (optional per the HQ contract)', () => { + const { thumb, ...noThumb } = BASE_ENTRY as any + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [noThumb])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(0) + }) + it('catches a missing entry key', () => { - const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'], url: null } // no "thumb" + const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'] } // no "url" writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) const result = run(['--check', '--file', 'wall.json'], dir) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/missing key\(s\) thumb/) + expect(result.stderr).toMatch(/missing key\(s\) url/) }) - it('catches an unexpected top-level key', () => { + it('catches an unexpected top-level key (e.g. the retired "history" field)', () => { const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) - raw.extra = 'not allowed' + raw.history = 'retired field' writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) const result = run(['--check', '--file', 'wall.json'], dir) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/unexpected key\(s\) extra/) + expect(result.stderr).toMatch(/unexpected key\(s\) history/) }) it('catches entries that are not newest-first', () => { From aa457d715937142607245f93437f987ba00248f0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:52:05 -0700 Subject: [PATCH 23/24] chore(releases): both walls leave the reference repo, RELEASES.md points home MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit releases/open-brainy.json follows brainy.json out โ€” the shared repo (soulcraftlabs/releases on The Source) is now the one home for both products' release notes; this repo hosts neither. The releases/ directory is gone. RELEASES.md gains a pointer, under the heading, to the two raw URLs HQ's /hq/releases door reads (this file stays as the human-readable quick reference; those files are the source of truth). --- RELEASES.md | 7 ++ releases/open-brainy.json | 136 -------------------------------------- 2 files changed, 7 insertions(+), 136 deletions(-) delete mode 100644 releases/open-brainy.json diff --git a/RELEASES.md b/RELEASES.md index e8833b80..c875cb26 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,5 +1,12 @@ # @soulcraft/brainy โ€” Release Notes for Consumers +Machine-readable release notes are published at +https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json +(this engine) and +https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json +(the product engine) โ€” read by HQ's `/hq/releases` door, and the source of +truth ahead of this file. + This file is the **quick reference for downstream sessions** tracking Brainy changes. Full auto-generated changelog: `CHANGELOG.md` ยท Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 9f1cd239..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.11", - "date": "2026-09-02", - "headline": "Hybrid finds filter before they hydrate, one owner per shutdown, and a faster open", - "items": [ - "Hybrid finds (query/vector combined with a filter, including connected and fusion finds) now filter first and hydrate only the page โ€” one batchGet of exactly the requested rows, instead of hydrating everything the search side found. Fixes a bug where any page after the first came back empty.", - "A brain now has exactly one shutdown owner โ€” a host and its engine no longer race to close the same store, and a follow-up flush requested during a running flush is handed off cleanly instead of ever risking a stall.", - "find({ path }) and other path-scoped VFS searches now serve a real range over the indexed path (O(log n)) instead of refusing the query outright โ€” both scoped and recursive:false searches were silently broken before this.", - "Open no longer rescans a brain's whole fact log on every open โ€” sealed segments the manifest already accounts for are skipped, collapsing a multi-second open term to near-zero on large brains.", - "commitTransaction() now refuses by name if single-ops are still pending, and a read-only open no longer writes clean-shutdown evidence it didn't earn โ€” two correctness invariants that were previously assumed, not enforced." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11", - "thumb": null - }, - { - "version": "10.4.10", - "date": "2026-09-02", - "headline": "A planner door for indexes, batched containment repair, and a fixed near()", - "items": [ - "An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.", - "repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.", - "find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10", - "thumb": null - }, - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows โ€” correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind โ€” four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open โ€” measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass โ€” parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index โ€” operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure โ€” never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy โ€” the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints โ€” a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} From 97b5ea2d5ddb739f1d1d0ff4e31664b5ef551df4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:59:49 -0700 Subject: [PATCH 24/24] =?UTF-8?q?fix(wall):=20every=20entry=20carries=20an?= =?UTF-8?q?=20https=20permalink=20=E2=80=94=20the=20product=20engine=20lin?= =?UTF-8?q?ks=20its=20public=20package=20page;=20null=20refused,=20an=20un?= =?UTF-8?q?known=20product=20refuses=20by=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/wall-entry.mjs | 27 ++++++++++++++++----------- tests/unit/release/wall-entry.test.ts | 16 +++++++++++++--- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs index 043341eb..d4ec7ba5 100644 --- a/scripts/wall-entry.mjs +++ b/scripts/wall-entry.mjs @@ -73,13 +73,14 @@ const ENTRY_OPTIONAL_KEYS = ['thumb'] const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS] const FILE_KEYS = ['product', 'entries'] -// The public release-page URL pattern, by product โ€” only products with a -// PUBLIC forge repo get a derived link. A product without an entry here -// (e.g. "brainy", whose repo is private) gets url: null, matching every -// entry the fleet has shipped for it so far โ€” a private link would 404 for -// anyone reading the public HQ page. +// The public permalink pattern, by product. Every entry MUST carry an https +// permalink: HQ's parser rejects a wall whose entries carry url: null (the +// whole feed became unreadable on 2026-09-02). A product whose forge repo is +// private links its PUBLIC package page on The Source instead of a release +// page that would 404 for HQ's readers. const RELEASE_URL_PATTERNS = { 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, + 'brainy': (version) => `https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/${version}`, } /** @@ -203,8 +204,8 @@ function validateShape(data) { if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) } - if (!(entry.url === null || typeof entry.url === 'string')) { - errors.push(`${label}: "url" must be a string or null`) + if (typeof entry.url !== 'string' || !/^https:\/\/\S+$/.test(entry.url)) { + errors.push(`${label}: "url" must be an https permalink โ€” never null; HQ's parser rejects the whole feed`) } if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) { errors.push(`${label}: "thumb" must be a string or null when present`) @@ -274,8 +275,8 @@ function extractChangelogBullets(changelog, version) { /** * Derive a wall entry from a CHANGELOG.md. - * @param {{product: string, version: string, date: string, changelogPath: string, url?: string | null, thumb?: string | null}} opts - * @returns {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} + * @param {{product: string, version: string, date: string, changelogPath: string, url?: string, thumb?: string | null}} opts + * @returns {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} */ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) @@ -288,7 +289,11 @@ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { const items = extractChangelogBullets(changelog, version) const headline = items[0] - const resolvedUrl = url !== undefined ? url : (RELEASE_URL_PATTERNS[product]?.(version) ?? null) + const pattern = RELEASE_URL_PATTERNS[product] + if (url === undefined && pattern === undefined) { + throw new Error(`wall-entry: no permalink pattern for product "${product}" โ€” add one to RELEASE_URL_PATTERNS or pass --url; entries never carry url: null`) + } + const resolvedUrl = url !== undefined ? url : pattern(version) const resolvedThumb = thumb !== undefined ? thumb : null return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } @@ -382,7 +387,7 @@ function ensureReleasesClone(remote, cacheDir) { * existing entry for the same version (idempotent re-runs), validating * before and after, committing, and pushing โ€” or refusing loudly, naming * the cure, at whichever step fails. - * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} entry * @param {string} product * @param {string} remote * @param {string} cacheDir diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts index 7f96da25..8bf9d357 100644 --- a/tests/unit/release/wall-entry.test.ts +++ b/tests/unit/release/wall-entry.test.ts @@ -192,8 +192,8 @@ describe('wall-entry.mjs โ€” generate + publish', () => { expect(readRemote(remoteDir, 'open-brainy')).toEqual(before) }) - it('derives no URL (null) for a product with no known public release-page pattern', () => { - seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }]) + it('derives the public package-page permalink for the product engine (private repo, never null)', () => { + seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: 'https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.5' }]) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) const result = run( @@ -203,10 +203,20 @@ describe('wall-entry.mjs โ€” generate + publish', () => { expect(result.status).toBe(0) const wall = readRemote(remoteDir, 'brainy') - expect(wall.entries[0].url).toBeNull() + expect(wall.entries[0].url).toBe('https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.6') expect(wall.entries[0].thumb).toBeNull() }) + it('refuses a product with no permalink pattern, naming the cure', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['feat: first'] }])) + + const result = run(['--product', 'mystery', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) + expect(result.status).not.toBe(0) + expect(result.stderr).toMatch(/no permalink pattern for product "mystery"/) + expect(result.stderr).toMatch(/never carry url: null/) + }) + it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => { seedRemote(remoteDir, 'open-brainy', []) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }]))