diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3–4× faster imports", - "items": [ - "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", - "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine — Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place — no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 582f4847..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} diff --git a/scripts/release.sh b/scripts/release.sh index 5d434320..67ad1053 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -248,12 +248,17 @@ else echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" fi -# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): -# soulcraft.com/docs carries the paid product's documentation only. This -# engine's documentation home is THIS repository — README and docs/ — and the -# site serves 301s for the slugs this rail used to push. The push script stays -# in the tree for history; the rail no longer calls it. -echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" +# Step 12: Push public docs to the soulcraft.com docs ingest door +# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when +# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — +# that already happened) when a push errors, so the docs site never +# silently trails npm. +echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" +if node scripts/push-docs.js; then + echo -e "${GREEN}✅ Docs push step done${NC}\n" +else + echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" +fi echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts index 91451281..0c14b60c 100644 --- a/src/db/generationSegments.ts +++ b/src/db/generationSegments.ts @@ -147,60 +147,6 @@ export class GenerationSegmentStore { return this.coveringSegment(gen) !== null } - /** - * @description True when `meta` declares more generations than it holds - * frames — a segment sealed by a writer that folded across a hole. The - * manifest records `frames` at fold time, so this is an O(1) comparison - * against the declared span and needs no I/O. - */ - private isSparse(meta: SegmentMeta): boolean { - return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames - } - - /** - * @description The generations this tier ACTUALLY holds, as coalesced - * ascending intervals — not what the segments declare. - * - * Dense segments (every one a current writer produces) contribute their - * declared range with no I/O. A SPARSE segment — one sealed before the - * density law was enforced, whose declared range spans generations it has - * no frame for — has its real generation list read from its sidecar and - * contributed instead, with the discrepancy narrated once. - * - * This is what keeps a store that already carries the damage from wedging. - * `open()` seeds `committedRanges` from these intervals, so a hole is never - * re-admitted as a committed generation, and the auto-compaction pass that - * used to fail on every run with "packed history is damaged" simply never - * asks for the missing frame. - * - * @returns Ascending, non-overlapping `[first, last]` intervals. - */ - async actualRanges(): Promise> { - const out: Array<[number, number]> = [] - for (const meta of this.manifest.segments) { - if (!this.isSparse(meta)) { - out.push([meta.firstGeneration, meta.lastGeneration]) - continue - } - const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames - prodLog.warn( - `[GenerationSegments] sealed segment ${meta.file} declares generations ` + - `${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` + - `frame(s) — ${missing} generation(s) in that span were never folded into it. ` + - `Serving the frames it actually holds; the declared span is not treated as ` + - `committed history. (Written by a pre-density-law writer that folded across a ` + - `gap; the segment itself is intact and no record is lost.)` - ) - const idx = await this.sidecarFor(meta) - for (const [gen] of idx.generations) { - const last = out[out.length - 1] - if (last !== undefined && gen === last[1] + 1) last[1] = gen - else out.push([gen, gen]) - } - } - return out - } - /** * Fold consecutive generations into ONE new sealed segment + sidecar and * append it to the manifest atomically. Caller guarantees: `gens` is @@ -218,38 +164,6 @@ export class GenerationSegmentStore { throw new Error('[GenerationSegments] fold() input must be strictly ascending') } } - // THE DENSITY LAW, MADE MECHANICAL. - // - // A sealed segment declares a CONTIGUOUS range [firstGeneration, - // lastGeneration] and every reader treats that range as containment: - // `coveringSegment` is an interval test, `hasGeneration` returns true for - // anything inside it, and `open()` seeds committedRanges from it. So a - // segment folded from a SPARSE input silently claims generations it does - // not hold, and the first read of one of those holes throws - // "inside sealed segment ... but has no frame — packed history is damaged". - // - // That is exactly how the damage was produced. `repackHistory` skipped - // generations mid-batch — ones absent from committedRanges, ones still in - // the pending buffer, ones whose tx.json would not read — and handed the - // survivors here, where the range was computed from the first and last of - // them. Worse, the mis-declared range was then merged back into - // committedRanges at the next open, which is what turned a quiet hole into - // a repeating auto-compaction failure on every subsequent run. - // - // Callers now split at discontinuities; this refusal is what keeps any - // future caller from reintroducing the class. A refusal here loses - // nothing — the generations stay in the live tier, readable, and the next - // pass folds them correctly. - for (let i = 1; i < gens.length; i++) { - if (gens[i].generation !== gens[i - 1].generation + 1) { - throw new Error( - `[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` + - `${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` + - `generation(s). A sealed segment declares a dense range, so folding a sparse ` + - `batch would claim generations it does not hold. Split the batch at the gap.` - ) - } - } const last = this.manifest.segments[this.manifest.segments.length - 1] if (last && gens[0].generation <= last.lastGeneration) { throw new Error( @@ -450,37 +364,12 @@ export class GenerationSegmentStore { return this.decodeFrame(payload) } } - // Inside the covering range but with no frame. Two very different causes, - // and conflating them is what made this class wedge every maintenance pass - // on the affected stores. - // - // (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than - // the span it declares. That segment was sealed by a writer that - // folded across a hole (the class this file's density law now bars). - // The segment is INTACT and nothing is lost; it simply never held this - // generation. Answering "not packed" is the honest answer, and it lets - // the caller's two-tier read decide what a genuinely absent generation - // means, instead of every compaction pass dying on a repeating throw. - // `actualRanges()` keeps such holes out of committedRanges at open, so - // in a healed store nobody asks this question in the first place. - // - // (2) A DENSE SEGMENT missing a frame it says it has — the manifest and - // the sidecar disagree about a segment that claims to be complete. - // That IS damage, and it stays loud. - if (this.isSparse(meta)) { - prodLog.warn( - `[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` + - `declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` + - `holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` + - `-generation span — it was sealed across a gap and never held this generation. ` + - `Reporting it as unpacked rather than as damage; no record is lost.` - ) - return null - } + // In the covering range but not present: the packed tier is dense by + // construction (fold packs every generation it is handed, including + // record-less ones) — absence inside a sealed range is damage. throw new Error( `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + - `span — the manifest and the sidecar disagree; packed history is damaged` + `range but has no frame — packed history is damaged` ) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index da21dc61..fd052c31 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -96,35 +96,6 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' -/** - * @description Split an ascending list of fold candidates into maximal - * CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`. - * - * A sealed segment declares one dense range `[firstGeneration, - * lastGeneration]`, and every reader treats that range as containment. So a - * batch with a hole in it must never become one segment: it would claim a - * generation it does not hold, and the first read of that hole reports the - * packed history as damaged. One run, one segment — the ranges then describe - * exactly what the segments contain. - * - * @param gens - Fold candidates, strictly ascending by generation. - * @returns One array per contiguous run, in ascending order. Empty in, empty out. - */ -export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] { - const runs: FoldGeneration[][] = [] - let run: FoldGeneration[] = [] - for (const g of gens) { - const prev = run[run.length - 1] - if (prev !== undefined && g.generation !== prev.generation + 1) { - runs.push(run) - run = [] - } - run.push(g) - } - if (run.length > 0) runs.push(run) - return runs -} - /** * @description Phases of the {@link GenerationStore.commitTransaction} commit * protocol at which a test-only fault injector can simulate a process crash. @@ -813,15 +784,9 @@ export class GenerationStore { if (storageSupportsFactLog(this.storage)) { this.segments = new GenerationSegmentStore(this.storage) await this.segments.open() - // ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law - // writer can declare a span wider than the frames it holds; seeding - // committedRanges from the declared span re-admits those holes as - // committed generations, and every later maintenance pass then asks for a - // frame that was never written. `actualRanges()` reads the real - // generation list from the sidecar for exactly those segments (and does - // no I/O for the dense ones, which is all of them on a healthy store). - const packedRanges = (await this.segments.actualRanges()) - .map((r): [number, number] => [r[0], Math.min(r[1], this.committed)]) + const packedRanges = this.segments + .segments() + .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) .filter(([lo, hi]) => lo <= hi) if (packedRanges.length > 0) { // Merge packed (older) + live (newer) interval sets — both ascending; @@ -3156,26 +3121,13 @@ export class GenerationStore { foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) } if (foldInput.length === 0) continue - // SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three - // filters above punch holes in it: a generation missing from - // committedRanges never appears, one still in the pending buffer is - // skipped, and one whose tx.json will not read is skipped. A sealed - // segment declares a DENSE range, so folding across such a hole makes - // the segment claim a generation it does not hold; the next open - // merges that mis-declared range into committedRanges, and every - // subsequent auto-compaction pass then asks for the missing frame and - // fails with "packed history is damaged". Fold each contiguous RUN as - // its own segment instead — same bytes, honest ranges. - for (const run of contiguousRuns(foldInput)) { - if (deadline !== undefined && Date.now() >= deadline) break - await segments.fold(run) - segmentsCreated++ - // Segment + manifest durable → the live copies retire. - for (const g of run) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) - } - folded += run.length + await segments.fold(foldInput) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of foldInput) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) } + folded += foldInput.length } if (folded > 0) { prodLog.info( diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts index bb07268d..2bcee038 100644 --- a/tests/integration/history-repacking.test.ts +++ b/tests/integration/history-repacking.test.ts @@ -16,7 +16,6 @@ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as path from 'node:path' import * as os from 'node:os' -import * as zlib from 'node:zlib' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { GenerationStore } from '../../src/db/generationStore.js' @@ -58,107 +57,6 @@ describe('history repacking — the two-tier lifecycle', () => { } }) - /** - * THE HOLE, END TO END — the shape a real store carries. - * - * A forensic fixture was measured with generation directories 1..2503 - * present except for exactly one: 1416. Its fact-log segment already showed - * the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration - * 1940 (531 generations) while recording only 530 facts. - * - * Before the fix, repacking such a store folded ACROSS that hole: the batch - * skipped 1416 (no readable delta) and the sealed segment declared a range - * spanning it anyway. The next open merged that declared range back into - * committedRanges, re-admitting 1416 as committed history, and every - * subsequent auto-compaction pass then asked the packed tier for a frame - * that was never written — producing, on EVERY run, the non-fatal narration - * - * Auto-compaction of generational history failed (non-fatal): generation - * N is inside sealed segment seg-....bgs's declared range but has no frame - * — packed history is damaged - * - * This pin removes a generation directory to make the same hole, then - * requires repack + reopen + compaction to complete cleanly. - */ - it('a missing generation directory does not poison the packed tier', async () => { - const dir = tempDir() - // `retention: 'all'` throughout: close() otherwise auto-compacts the - // history away, and this pin needs the cold generations still on disk so - // there is something to punch a hole in. The live window stays at its - // production default for the build phase, so nothing folds yet. - const archival = async (): Promise => { - const b = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - embeddingFunction: stub, - retention: 'all' - }) - await b.init() - return b - } - const brain = await archival() - - const id = await brain.add({ - data: 'holed-entity', - type: NounType.Document, - metadata: { v: 0 } - }) - // One flush per update: single-op writes coalesce inside a flush window, - // so a history deep enough to have a middle needs the windows separated. - for (let v = 1; v <= 12; v++) { - await brain.update({ id, metadata: { v } }) - await brain.flush() - } - await brain.close() - - // Punch the hole: delete ONE generation directory in the middle of the - // cold range, exactly as the real store presents it. - const genRoot = path.join(dir, '_generations') - const numeric = fs - .readdirSync(genRoot, { withFileTypes: true }) - .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)) - .map((e) => Number(e.name)) - .sort((a, b) => a - b) - expect(numeric.length).toBeGreaterThan(6) - const victim = numeric[Math.floor(numeric.length / 2)] - fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true }) - - // Now shrink the live window and reopen. close() repacks automatically - // (brainy.ts phase 0b), so this is the production sequence exactly: a - // store with a hole in its history gets folded by ordinary housekeeping, - // with nobody asking for it. - ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 - const reopened = await archival() - const result = await reopened.repackHistory() - expect(result.foldedGenerations).toBeGreaterThan(0) - - const segDir = path.join(dir, SEGMENTS_PREFIX) - const manifestPath = ['manifest.json', 'manifest.json.gz'] - .map((f) => path.join(segDir, f)) - .find((p) => fs.existsSync(p))! - const raw = manifestPath.endsWith('.gz') - ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8') - : fs.readFileSync(manifestPath, 'utf8') - const manifest = JSON.parse(raw) as { - segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }> - } - - // THE LAW: every sealed segment declares exactly as many generations as it - // holds frames, and none of them spans the victim. - for (const s of manifest.segments) { - expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames) - expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false) - } - - await reopened.close() - - // And the pass that used to fail on every run now completes: reopen (which - // re-seeds committedRanges from the packed tier) then compact history. - const third = await openBrain(dir) - await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined() - await third.close() - }) - it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 const dir = tempDir() diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts index f16e67b3..27ab85cb 100644 --- a/tests/unit/db/generation-segments.test.ts +++ b/tests/unit/db/generation-segments.test.ts @@ -147,119 +147,4 @@ describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) - - // ========================================================================== - // THE DENSITY LAW - // ========================================================================== - // - // A sealed segment declares a CONTIGUOUS range and every reader treats that - // range as containment. Folding a sparse batch therefore makes the segment - // claim generations it does not hold — and because `open()` merges declared - // ranges back into committedRanges, the hole is re-admitted as committed - // history and every later maintenance pass fails asking for a frame that was - // never written. That is the "generation N is inside sealed segment - // seg-....bgs's declared range but has no frame — packed history is damaged" - // narration seen on every run of the affected stores. - - it('fold REFUSES a batch with a hole — a dense range may not be declared over sparse input', async () => { - await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow( - /not contiguous: 2 → 4 skips 1 generation/ - ) - // The refusal loses nothing: no segment was sealed, so the generations - // stay in the live tier and the next pass folds them correctly. - expect(store.segments()).toHaveLength(0) - expect(store.hasGeneration(1)).toBe(false) - }) - - it('a wider gap names how many generations it would have swallowed', async () => { - await expect(store.fold([gen(10), gen(20)])).rejects.toThrow( - /not contiguous: 10 → 20 skips 9 generation\(s\)/ - ) - }) - - it('two contiguous runs folded separately declare honest ranges', async () => { - // What the caller now does instead of folding across the gap. - const a = await store.fold([gen(1), gen(2), gen(3)]) - const b = await store.fold([gen(7), gen(8)]) - expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) - expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 }) - // The gap is honestly outside the packed tier. - for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false) - for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true) - expect(await store.actualRanges()).toEqual([ - [1, 3], - [7, 8] - ]) - }) - - it('actualRanges() is exact and I/O-free for dense segments', async () => { - await store.fold([gen(1), gen(2)]) - await store.fold([gen(3), gen(4)]) - // Adjacent dense segments each contribute their declared range. - expect(await store.actualRanges()).toEqual([ - [1, 2], - [3, 4] - ]) - }) - - // ---- pre-existing damage: a store sealed by the old writer ---------------- - - /** - * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and - * sidecar for a contiguous run, then rewrite the manifest so the segment - * declares a wider range than the frames it holds. This reproduces on disk - * exactly what the affected stores carry, without needing the old code. - */ - const sealSparseSegment = async (): Promise => { - await store.fold([gen(1), gen(2), gen(3)]) - const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any - // Declare 1..5 while holding frames for 1..3 — generations 4 and 5 become - // holes inside a sealed range. - manifest.segments[0].lastGeneration = 5 - await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest) - } - - it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => { - await sealSparseSegment() - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - - // The frames it really holds still serve, byte-faithfully. - expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002) - expect(await reopened.readRecords(3)).toHaveLength(2) - - // The holes answer "not packed" instead of throwing. This is the fix for - // the wedge: the old reader threw here on EVERY maintenance pass. - expect(await reopened.readDelta(4)).toBeNull() - expect(await reopened.readRecords(5)).toBeNull() - }) - - it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => { - await sealSparseSegment() - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - // Declared 1..5; actually holds 1..3. The store seeds committedRanges from - // THIS, so generations 4 and 5 never become committed history again. - expect(await reopened.actualRanges()).toEqual([[1, 3]]) - }) - - it('a DENSE segment missing a frame is still loud damage', async () => { - // The other side of the branch: when the manifest claims a complete span, - // a missing frame means the manifest and sidecar disagree — real damage, - // and it must not be quietly downgraded to "unpacked". - await store.fold([gen(1), gen(2), gen(3)]) - const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` - const raw = (await storage.readRawBytes(idxPath))! - const { decode, encode } = await import('@msgpack/msgpack') - const idx = decode(raw) as any - // Drop generation 2's entry while the manifest still declares 3 frames. - idx.generations = idx.generations.filter(([g]: [number]) => g !== 2) - await storage.writeRawBytes(idxPath, encode(idx)) - - const reopened = new GenerationSegmentStore(storage as any) - await reopened.open() - await expect(reopened.readDelta(2)).rejects.toThrow( - /manifest and the sidecar disagree; packed history is damaged/ - ) - }) })