From f8e6da2b6603e52e12ea35983c4e7a122921d790 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 19 Jul 2026 14:54:36 -0700 Subject: [PATCH 01/10] =?UTF-8?q?feat:=20scanFacts=20liveness=20contract?= =?UTF-8?q?=20=E2=80=94=20first=20batch=20or=20loud=20failure=20within=20a?= =?UTF-8?q?=20documented=20bound?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage-2 D1 contract item (co-frozen): a fact scan may be slow, never silent. batches() now races its FIRST pull against SCANFACTS_FIRST_BATCH_MS (10s, exported; test-overridable) — a wedged or unreadably slow store produces a loud abort naming the contract instead of a consumer hanging indistinguishably from progress (the production shape: a heal against a generations-backlogged brain wedged silently on the first segment read). Only the first pull is raced: the bound is time-to-first-batch (proof the producer is alive), not per-batch pacing, and it runs only while a pull is pending — consumer think-time between pulls never counts against the producer (pinned). Three pins: wedged-store loud failure within the bound, healthy scan untouched end-to-end, slow-consumer immunity. --- src/db/factLog.ts | 52 ++++++++++++++++++++++++++++++++-- src/index.ts | 1 + tests/unit/db/fact-log.test.ts | 48 +++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 2 deletions(-) diff --git a/src/db/factLog.ts b/src/db/factLog.ts index 4c5e95fd..94e79700 100644 --- a/src/db/factLog.ts +++ b/src/db/factLog.ts @@ -102,12 +102,26 @@ export interface FactScanBatch { segmentId: string } +/** + * Liveness bound on a scan's FIRST batch (Stage-2 co-freeze, D1 contract): + * `batches()` must yield its first batch — or fail loudly — within this many + * ms of the first pull. A backlogged or damaged store may be SLOW, but it may + * never be SILENT: a consumer awaiting the first batch is otherwise + * indistinguishable from a wedge (the exact failure shape a production heal + * hit against a generations-backlogged brain). + */ +export const SCANFACTS_FIRST_BATCH_MS = 10_000 + /** The telemetry a scan OPEN returns (frozen shape). */ export interface FactScanHandle { headGeneration: number segmentCount: number approxFactCount: number - /** Ordered batches; a detected gap aborts LOUDLY, never a silent skip. */ + /** + * Ordered batches; a detected gap aborts LOUDLY, never a silent skip. + * Liveness contract: the FIRST batch resolves or rejects within + * {@link SCANFACTS_FIRST_BATCH_MS} of the first pull — never a silent hang. + */ batches: () => AsyncGenerator /** Close telemetry — the invariant cross-check, valid after iteration ends. */ summary: () => { factsYielded: number; segmentsRead: number } @@ -440,6 +454,8 @@ export class FactLog { toGeneration?: number kinds?: Array<'noun' | 'verb'> batchSize?: number + /** Test override for the first-batch liveness bound (default {@link SCANFACTS_FIRST_BATCH_MS}). */ + firstBatchTimeoutMs?: number }): FactScanHandle { const from = options?.fromGeneration ?? 1 const to = options?.toGeneration ?? this.head @@ -514,11 +530,43 @@ export class FactLog { } } + // Liveness wrapper: the FIRST pull races the contract deadline. Only the + // first — the bound is time-to-first-batch (proof the producer is alive), + // not per-batch pacing; and it runs only while a pull is actually pending, + // so consumer think-time between pulls never counts against the producer. + const firstBatchTimeoutMs = options?.firstBatchTimeoutMs ?? SCANFACTS_FIRST_BATCH_MS + async function* batchesWithLiveness(this: void): AsyncGenerator { + const inner = batches() + let timer: NodeJS.Timeout | undefined + try { + const deadline = new Promise((_, reject) => { + timer = setTimeout( + () => + reject( + new Error( + `fact log: scanFacts produced no first batch within ${firstBatchTimeoutMs}ms ` + + `(liveness contract) — the store is wedged or unreadably slow; aborting scan LOUDLY ` + + `instead of hanging the consumer.` + ) + ), + firstBatchTimeoutMs + ) + timer.unref?.() + }) + const first = await Promise.race([inner.next(), deadline]) + if (first.done) return + yield first.value + } finally { + clearTimeout(timer) + } + yield* inner + } + return { headGeneration: this.head, segmentCount: segments.length + (tailSnapshot.length > 0 ? 1 : 0), approxFactCount, - batches, + batches: batchesWithLiveness, summary: () => ({ factsYielded, segmentsRead }) } } diff --git a/src/index.ts b/src/index.ts index ee01d885..b978b9fd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -213,6 +213,7 @@ export type { CommitFact, FactOp, FactScanBatch, + SCANFACTS_FIRST_BATCH_MS, FactScanHandle } from './db/factLog.js' // The generalized family stamp — which source generation a projection diff --git a/tests/unit/db/fact-log.test.ts b/tests/unit/db/fact-log.test.ts index abce2dc9..f1c226cc 100644 --- a/tests/unit/db/fact-log.test.ts +++ b/tests/unit/db/fact-log.test.ts @@ -186,4 +186,52 @@ describe('fact log — round-trip, framing, reconcile, rotation, scan', () => { await log.sync() expect(log.segmentPaths()).toEqual([]) // only a tail exists — nothing sealed }) + + describe('scanFacts liveness contract (Stage-2 D1)', () => { + it('a wedged store fails LOUDLY within the first-batch bound — never a silent hang', async () => { + // Force a sealed segment (tiny rotateBytes) so the scan must READ from + // storage, then wedge that read: the exact production shape (a + // backlogged brain whose segment read never returned). + const mem: any = new MemoryStorage() + await mem.init() + const wedgeable = new FactLog(mem, { rotateBytes: 1 }) + await wedgeable.open(0) + await wedgeable.append(fact(1)) + await wedgeable.append(fact(2)) // second append rotates → seg 1 sealed + await wedgeable.sync() + + const realRead = mem.readRawBytes.bind(mem) + mem.readRawBytes = (p: string) => + p.includes('facts/seg-') ? new Promise(() => {}) : realRead(p) // hangs forever + + const scan = wedgeable.scanFacts({ firstBatchTimeoutMs: 200 }) + const started = Date.now() + await expect(scan.batches().next()).rejects.toThrow(/no first batch within 200ms/) + expect(Date.now() - started).toBeLessThan(5_000) // bound held, not a hang + }) + + it('a healthy scan is unaffected — first batch well inside the bound, all facts delivered', async () => { + for (let g = 1; g <= 5; g++) await log.append(fact(g)) + await log.sync() + const scan = log.scanFacts({ batchSize: 2 }) + const all: CommitFact[] = [] + for await (const b of scan.batches()) all.push(...b.facts) + expect(all.map((f) => f.generation)).toEqual([1, 2, 3, 4, 5]) + expect(scan.summary().factsYielded).toBe(5) + }) + + it('consumer think-time between pulls never counts against the producer', async () => { + for (let g = 1; g <= 4; g++) await log.append(fact(g)) + await log.sync() + // Bound tighter than the consumer's pause: only the FIRST pull is + // raced, so a slow consumer after batch 1 must not trip the deadline. + const gen = log.scanFacts({ batchSize: 2, firstBatchTimeoutMs: 150 }).batches() + const first = await gen.next() + expect(first.done).toBe(false) + await new Promise((r) => setTimeout(r, 400)) // dawdle past the bound + const second = await gen.next() + expect(second.done).toBe(false) + expect((await gen.next()).done).toBe(true) + }) + }) }) From d8acb3776b2e64db79332cb70fb3b0d7588cef99 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 19 Jul 2026 15:14:27 -0700 Subject: [PATCH 02/10] =?UTF-8?q?feat:=20generation-segment=20store=20?= =?UTF-8?q?=E2=80=94=20the=20D1+D3=20packed-tier=20file=20format?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First stage of the co-frozen D1+D3+repacking unit: the format core, self-contained under _generations/segments/. - seg-.bgs: append-once packs of consecutive generations (magic BGS1; frame = u32 len + u32 crc32c + msgpack [generation, timestamp, delta, records, flags]; flags reserves compressed-payload evolution without a format break). Sealed segments are immutable — fold refuses overlap with sealed ranges. - seg-.idx: DERIVED sidecar (per-generation frame offsets + per-id generation postings + checksums); lost/corrupt sidecars rebuild from their segment loudly; a damaged segment (frame CRC mismatch) fails loudly, never serves wrong bytes. - manifest.json: the one discovery path — open() reads it and never lists the packed backlog (the scan-wedge class's cure); refuses a newer manifest version rather than serving partial history. - D3 semantics: dropSegmentsBelow reclaims WHOLE segments at boundaries only and bumps compactedBelow durably; archival-profile enforcement stays with the caller per the co-freeze. - D8 rider: digestThroughPacked(g) — deterministic crc32c chain over sealed-segment checksums (+ frame-level prefix mid-segment), O(segments), reopen-stable. Also fixes a cross-adapter contract bug the suite caught: memory storage's deleteObjectFromPath ignored the raw-bytes store, so deleteRawObject on a raw-bytes path (fact-log or segment files) silently no-op'd — deletes now match filesystem unlink semantics. Six pins. Wiring into GenerationStore (two-tier reads, the repacker, cold-open manifest path) lands with the rest of the unit before its release; cortex's fact-record/stamp shapes reconcile the sidecar keying when they post. --- src/db/generationSegments.ts | 459 ++++++++++++++++++++++ src/storage/adapters/memoryStorage.ts | 5 + tests/unit/db/generation-segments.test.ts | 150 +++++++ 3 files changed, 614 insertions(+) create mode 100644 src/db/generationSegments.ts create mode 100644 tests/unit/db/generation-segments.test.ts diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts new file mode 100644 index 00000000..0c14b60c --- /dev/null +++ b/src/db/generationSegments.ts @@ -0,0 +1,459 @@ +/** + * @module db/generationSegments + * @description The generation-segment store — Stage-2 D1+D3+repacking's file + * format (co-frozen 2026-07-19; design: the d1-d3-repacking spec). + * + * Packs CONSECUTIVE cold generations' record-sets (before-images + delta) + * into append-once segment files with derived sidecar indexes, so history + * scales in SEGMENTS (tens) instead of FILES-PER-GENERATION (hundreds of + * thousands), and cold-open reads ONE manifest instead of listing the + * backlog. Layout under `_generations/segments/`: + * + * - `seg-.bgs` — magic "BGS1", then one frame per + * generation: `u32 payloadLen | u32 crc32c | msgpack payload`. Payload is + * POSITIONAL: `[generation, timestamp, delta, records[], flags]` with + * records `[kindByte, id, record]`. `flags` reserves encoding evolution + * (bit 0 = compressed payload — v1 always 0; a future writer upgrade, + * never a format break). Sealed segments are IMMUTABLE — the fact log's + * own law, generalized. + * - `seg-.idx` — DERIVED sidecar (msgpack): per-generation frame + * offsets (point reads = one ranged read, never a listing) + per-id + * generation postings (per-id chain rebuilds read only what they need). + * Corrupt/missing → rebuilt from its segment in one sequential read, + * loudly. + * - `manifest.json` — the segment catalogue + `compactedBelow` (D3's + * horizon marker). Cold-open reads THIS; the packed backlog is never + * listed. + * + * D3 semantics carried here: bounded-retention reclaim drops WHOLE segments + * at boundaries (O(1) per segment, no rewrite); under the archival profile + * (`retention: 'all'`) nothing here is ever dropped — folding is the only + * transform (re-representation, never deletion). + */ + +import { encode as msgpackEncode, decode as msgpackDecode } from '@msgpack/msgpack' +import { crc32c } from '../utils/crc32c.js' +import type { FactLogStorage } from './factLog.js' +import { prodLog } from '../utils/logger.js' + +/** Directory for segment files + manifest, under the generations prefix. */ +export const SEGMENTS_PREFIX = '_generations/segments' + +/** Target sealed-segment size (co-freeze proposal; tunable on evidence). */ +export const SEGMENT_TARGET_BYTES = 64 * 1024 * 1024 + +const MAGIC = new TextEncoder().encode('BGS1') +const FRAME_PREFIX_BYTES = 8 // u32 payloadLen + u32 crc32c +const MANIFEST_PATH = `${SEGMENTS_PREFIX}/manifest.json` + +/** One generation's fold input — exactly what the live tier holds for it. */ +export interface FoldGeneration { + generation: number + timestamp: number + /** The tx.json delta object, carried verbatim. */ + delta: unknown + /** The before-image record-set (empty for record-less generations). */ + records: Array<{ kind: 'noun' | 'verb'; id: string; record: unknown }> +} + +/** Manifest entry for one sealed segment. */ +export interface SegmentMeta { + file: string + firstGeneration: number + lastGeneration: number + frames: number + bytes: number + /** crc32c of the full segment byte stream — the digest chain's link. */ + checksum: number +} + +interface SegmentManifest { + version: 1 + compactedBelow: number + segments: SegmentMeta[] +} + +interface SidecarIndex { + version: 1 + /** [generation, frameOffset, frameLen] ascending by generation. */ + generations: Array<[number, number, number]> + /** `${kindByte}:${id}` → ascending generations holding a record for it. */ + ids: Record +} + +const segmentFileName = (firstGeneration: number): string => + `seg-${String(firstGeneration).padStart(20, '0')}.bgs` +const sidecarFileName = (firstGeneration: number): string => + `seg-${String(firstGeneration).padStart(20, '0')}.idx` + +/** + * The generation-segment store. Owns the packed tier ONLY — the live + * per-generation tier and the routing between tiers belong to + * `GenerationStore`. All mutating entry points here are called under the + * generation store's commit mutex. + */ +export class GenerationSegmentStore { + private readonly storage: FactLogStorage + private manifest: SegmentManifest = { version: 1, compactedBelow: 0, segments: [] } + /** Sidecar cache — segments are immutable, so entries never invalidate. */ + private readonly sidecars = new Map() + + constructor(storage: FactLogStorage) { + this.storage = storage + } + + /** Load the manifest (ONE read — never a directory listing). */ + async open(): Promise { + const raw = (await this.storage.readRawObject(MANIFEST_PATH)) as SegmentManifest | null + if (raw) { + if (raw.version !== 1) { + throw new Error( + `[GenerationSegments] manifest version ${String(raw.version)} is newer than this ` + + `engine understands — refusing to serve partial history. Upgrade the engine.` + ) + } + this.manifest = raw + } + } + + /** The packed tier's catalogue (ascending, immutable snapshot). */ + segments(): readonly SegmentMeta[] { + return this.manifest.segments + } + + /** D3's horizon marker: generations below this were reclaimed (bounded profiles only). */ + compactedBelow(): number { + return this.manifest.compactedBelow + } + + /** The covering sealed segment for `gen`, or null if it lives outside the packed tier. */ + private coveringSegment(gen: number): SegmentMeta | null { + // Manifest is ascending and ranges never overlap — binary search. + const segs = this.manifest.segments + let lo = 0 + let hi = segs.length - 1 + while (lo <= hi) { + const mid = (lo + hi) >> 1 + const s = segs[mid] + if (gen < s.firstGeneration) hi = mid - 1 + else if (gen > s.lastGeneration) lo = mid + 1 + else return s + } + return null + } + + /** True when `gen` is packed (readable from this tier). */ + hasGeneration(gen: number): boolean { + return this.coveringSegment(gen) !== null + } + + /** + * Fold consecutive generations into ONE new sealed segment + sidecar and + * append it to the manifest atomically. Caller guarantees: `gens` is + * ascending, contiguous with the packed tier (first = last packed + 1 when + * segments exist), and already durable in the live tier. Crash between the + * segment write and the caller's live-tier delete leaves a DUPLICATE + * representation — resolved live-tier-wins by the reader; never a gap. + */ + async fold(gens: FoldGeneration[]): Promise { + if (gens.length === 0) { + throw new Error('[GenerationSegments] fold() requires at least one generation') + } + for (let i = 1; i < gens.length; i++) { + if (gens[i].generation <= gens[i - 1].generation) { + throw new Error('[GenerationSegments] fold() input must be strictly ascending') + } + } + const last = this.manifest.segments[this.manifest.segments.length - 1] + if (last && gens[0].generation <= last.lastGeneration) { + throw new Error( + `[GenerationSegments] fold() overlaps the packed tier: ${gens[0].generation} ≤ ` + + `sealed ${last.lastGeneration} — segments are immutable, never rewritten` + ) + } + + const first = gens[0].generation + const file = segmentFileName(first) + const sidecar: SidecarIndex = { version: 1, generations: [], ids: {} } + + // Encode all frames, tracking offsets for the sidecar. + const parts: Uint8Array[] = [MAGIC] + let offset = MAGIC.length + for (const g of gens) { + const payload = msgpackEncode([ + g.generation, + g.timestamp, + g.delta, + g.records.map((r) => [r.kind === 'noun' ? 0 : 1, r.id, r.record]), + 0 // flags: v1 = uncompressed + ]) + const frame = new Uint8Array(FRAME_PREFIX_BYTES + payload.length) + const view = new DataView(frame.buffer) + view.setUint32(0, payload.length, true) + view.setUint32(4, crc32c(payload), true) + frame.set(payload, FRAME_PREFIX_BYTES) + sidecar.generations.push([g.generation, offset, frame.length]) + for (const r of g.records) { + const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}` + ;(sidecar.ids[key] ??= []).push(g.generation) + } + parts.push(frame) + offset += frame.length + } + const total = parts.reduce((n, p) => n + p.length, 0) + const bytes = new Uint8Array(total) + let at = 0 + for (const p of parts) { + bytes.set(p, at) + at += p.length + } + + const meta: SegmentMeta = { + file, + firstGeneration: first, + lastGeneration: gens[gens.length - 1].generation, + frames: gens.length, + bytes: total, + checksum: crc32c(bytes) + } + + // Durability order: segment + sidecar fsync'd BEFORE the manifest names + // them (a crash before the manifest = invisible orphan files, harmless); + // manifest last, atomically. + const segPath = `${SEGMENTS_PREFIX}/${file}` + const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(first)}` + await this.storage.writeRawBytes(segPath, bytes) + await this.storage.writeRawBytes(idxPath, msgpackEncode(sidecar)) + await this.storage.syncRawObjects([segPath, idxPath]) + const next: SegmentManifest = { + ...this.manifest, + segments: [...this.manifest.segments, meta] + } + await this.storage.writeRawObject(MANIFEST_PATH, next) + await this.storage.syncRawObjects([MANIFEST_PATH]) + this.manifest = next + this.sidecars.set(file, sidecar) + return meta + } + + /** Load (or rebuild, loudly) a segment's sidecar. */ + private async sidecarFor(meta: SegmentMeta): Promise { + const cached = this.sidecars.get(meta.file) + if (cached) return cached + const idxPath = `${SEGMENTS_PREFIX}/${sidecarFileName(meta.firstGeneration)}` + const raw = await this.storage.readRawBytes(idxPath) + if (raw) { + try { + const idx = msgpackDecode(raw) as SidecarIndex + if (idx.version === 1) { + this.sidecars.set(meta.file, idx) + return idx + } + } catch { + // fall through to rebuild + } + } + // Sidecars are DERIVED: rebuild from the segment, loudly — never serve + // wrong offsets silently. + prodLog.warn( + `[GenerationSegments] sidecar for ${meta.file} missing or unreadable — rebuilding from the segment` + ) + const rebuilt = await this.rebuildSidecar(meta) + await this.storage.writeRawBytes(idxPath, msgpackEncode(rebuilt)) + this.sidecars.set(meta.file, rebuilt) + return rebuilt + } + + /** One sequential read of the segment → a fresh sidecar. Verifies every frame CRC. */ + private async rebuildSidecar(meta: SegmentMeta): Promise { + const frames = await this.readAllFrames(meta) + const idx: SidecarIndex = { version: 1, generations: [], ids: {} } + for (const f of frames) { + idx.generations.push([f.generation, f.offset, f.frameLen]) + for (const r of f.records) { + const key = `${r.kind === 'noun' ? 0 : 1}:${r.id}` + ;(idx.ids[key] ??= []).push(f.generation) + } + } + return idx + } + + private decodeFrame( + payload: Uint8Array + ): { generation: number; timestamp: number; delta: unknown; records: FoldGeneration['records'] } { + const [generation, timestamp, delta, rawRecords] = msgpackDecode(payload) as [ + number, + number, + unknown, + Array<[number, string, unknown]>, + number + ] + return { + generation, + timestamp, + delta, + records: rawRecords.map(([kindByte, id, record]) => ({ + kind: kindByte === 0 ? ('noun' as const) : ('verb' as const), + id, + record + })) + } + } + + private async readAllFrames(meta: SegmentMeta): Promise< + Array & { offset: number; frameLen: number }> + > { + const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`) + if (!bytes) { + throw new Error( + `[GenerationSegments] sealed segment ${meta.file} is MISSING — packed history is damaged; ` + + `refusing to continue silently` + ) + } + const out: Array & { offset: number; frameLen: number }> = [] + let at = MAGIC.length + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + while (at + FRAME_PREFIX_BYTES <= bytes.length) { + const payloadLen = view.getUint32(at, true) + const crc = view.getUint32(at + 4, true) + const payload = bytes.subarray(at + FRAME_PREFIX_BYTES, at + FRAME_PREFIX_BYTES + payloadLen) + if (payload.length !== payloadLen || crc32c(payload) !== crc) { + throw new Error( + `[GenerationSegments] frame CRC mismatch in ${meta.file} at offset ${at} — ` + + `packed history is damaged; refusing to serve it` + ) + } + out.push({ ...this.decodeFrame(payload), offset: at, frameLen: FRAME_PREFIX_BYTES + payloadLen }) + at += FRAME_PREFIX_BYTES + payloadLen + } + return out + } + + /** Read one packed generation's frame via its sidecar offset (one ranged read). */ + private async readFrame( + gen: number + ): Promise | null> { + const meta = this.coveringSegment(gen) + if (!meta) return null + const idx = await this.sidecarFor(meta) + // generations ascending → binary search. + const gens = idx.generations + let lo = 0 + let hi = gens.length - 1 + while (lo <= hi) { + const mid = (lo + hi) >> 1 + if (gens[mid][0] < gen) lo = mid + 1 + else if (gens[mid][0] > gen) hi = mid - 1 + else { + const [, offset, frameLen] = gens[mid] + const bytes = await this.storage.readRawBytes(`${SEGMENTS_PREFIX}/${meta.file}`) + if (!bytes) { + throw new Error(`[GenerationSegments] sealed segment ${meta.file} is MISSING`) + } + const frame = bytes.subarray(offset, offset + frameLen) + const view = new DataView(frame.buffer, frame.byteOffset, frame.byteLength) + const payloadLen = view.getUint32(0, true) + const crc = view.getUint32(4, true) + const payload = frame.subarray(FRAME_PREFIX_BYTES, FRAME_PREFIX_BYTES + payloadLen) + if (payload.length !== payloadLen || crc32c(payload) !== crc) { + throw new Error( + `[GenerationSegments] frame CRC mismatch for generation ${gen} in ${meta.file} — ` + + `packed history is damaged; refusing to serve it` + ) + } + return this.decodeFrame(payload) + } + } + // In the covering range but not present: the packed tier is dense by + // construction (fold packs every generation it is handed, including + // record-less ones) — absence inside a sealed range is damage. + throw new Error( + `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + + `range but has no frame — packed history is damaged` + ) + } + + /** The packed tier's delta for `gen` (null = not packed). */ + async readDelta(gen: number): Promise<{ delta: unknown; timestamp: number } | null> { + const frame = await this.readFrame(gen) + return frame ? { delta: frame.delta, timestamp: frame.timestamp } : null + } + + /** The packed tier's full record-set for `gen` (null = not packed). */ + async readRecords(gen: number): Promise { + const frame = await this.readFrame(gen) + return frame ? frame.records : null + } + + /** One packed before-image (null = not packed OR no record for the id in that generation). */ + async readRecord(gen: number, kind: 'noun' | 'verb', id: string): Promise { + const frame = await this.readFrame(gen) + if (!frame) return null + const hit = frame.records.find((r) => r.kind === kind && r.id === id) + return hit ? hit.record : null + } + + /** + * D3 reclaim: drop WHOLE segments whose lastGeneration < `belowGeneration` + * and bump `compactedBelow`. Partial segments are never dropped — the + * boundary waits. NEVER called under the archival profile (the caller + * enforces retention semantics; this method only executes boundary drops). + */ + async dropSegmentsBelow(belowGeneration: number): Promise<{ dropped: number; compactedBelow: number }> { + const keep: SegmentMeta[] = [] + const drop: SegmentMeta[] = [] + for (const s of this.manifest.segments) { + ;(s.lastGeneration < belowGeneration ? drop : keep).push(s) + } + if (drop.length === 0) { + return { dropped: 0, compactedBelow: this.manifest.compactedBelow } + } + const compactedBelow = Math.max( + this.manifest.compactedBelow, + drop[drop.length - 1].lastGeneration + 1 + ) + // Manifest first (the drop is authoritative once named), then bytes — + // a crash between leaves orphan segment files invisible to the manifest, + // harmless and re-collectable. + const next: SegmentManifest = { ...this.manifest, compactedBelow, segments: keep } + await this.storage.writeRawObject(MANIFEST_PATH, next) + await this.storage.syncRawObjects([MANIFEST_PATH]) + this.manifest = next + for (const s of drop) { + await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${s.file}`) + await this.storage.deleteRawObject(`${SEGMENTS_PREFIX}/${sidecarFileName(s.firstGeneration)}`) + this.sidecars.delete(s.file) + } + return { dropped: drop.length, compactedBelow } + } + + /** + * D8 rider — the packed portion of `generationDigest(g)`: a deterministic + * crc32c chain over sealed-segment checksums fully below `g`, plus the + * frame CRC of `g`'s own frame when `g` is mid-segment. O(segments), not + * O(generations); identical history ⇒ identical digest on any machine. + * The live-tier portion is composed by the caller. + */ + async digestThroughPacked(g: number): Promise { + let digest = 0 + let covered = false + for (const s of this.manifest.segments) { + if (s.lastGeneration <= g) { + digest = crc32c(new TextEncoder().encode(`${digest}:${s.checksum}`)) + if (s.lastGeneration === g) covered = true + } else if (s.firstGeneration <= g) { + // g is mid-segment: chain the partial prefix via g's frame CRC. + const frame = await this.readFrame(g) + if (frame === null) return null + const idx = await this.sidecarFor(s) + const upTo = idx.generations.filter(([gen]) => gen <= g) + for (const [gen, offset, frameLen] of upTo) { + digest = crc32c(new TextEncoder().encode(`${digest}:${gen}:${offset}:${frameLen}`)) + } + covered = true + break + } + } + return covered || this.manifest.segments.length > 0 ? digest : null + } +} diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index bab9d4d9..1b1f412e 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -133,6 +133,11 @@ export class MemoryStorage extends BaseStorage { */ protected async deleteObjectFromPath(path: string): Promise { this.objectStore.delete(path) + // Filesystem parity: on disk, objects and raw BYTE files are both just + // files — unlink removes whichever exists. Without this, deleteRawObject + // on a raw-bytes path (fact-log/generation segments) silently no-ops on + // memory storage: the delete "succeeds" and the bytes remain. + this.rawBytesStore.delete(path) } /** diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts new file mode 100644 index 00000000..27ab85cb --- /dev/null +++ b/tests/unit/db/generation-segments.test.ts @@ -0,0 +1,150 @@ +/** + * @module tests/unit/db/generation-segments + * @description The generation-segment store (Stage-2 D1+D3 file format). + * Laws: (1) fold → read round-trips deltas and records byte-faithfully via + * sidecar point-reads; (2) the manifest is the ONLY discovery path — reopen + * reads one file, never a listing; (3) a lost/corrupt sidecar rebuilds from + * its segment loudly, a damaged SEGMENT fails loudly (never silent wrong + * data); (4) D3 reclaim drops whole segments only and bumps compactedBelow; + * (5) the packed digest is deterministic across reopen; (6) immutability — + * fold refuses overlap with sealed ranges. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { + GenerationSegmentStore, + SEGMENTS_PREFIX, + type FoldGeneration +} from '../../../src/db/generationSegments.js' + +const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + +const gen = (g: number, recordCount = 2): FoldGeneration => ({ + generation: g, + timestamp: 1_700_000_000_000 + g, + delta: { generation: g, nouns: [UUID(g)], verbs: [], bytes: 123 + g }, + records: Array.from({ length: recordCount }, (_, i) => ({ + kind: (i % 2 === 0 ? 'noun' : 'verb') as 'noun' | 'verb', + id: UUID(g * 100 + i), + record: { metadata: { noun: 'document', v: g }, vector: { v: [g, i] } } + })) +}) + +describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { + let storage: MemoryStorage + let store: GenerationSegmentStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + store = new GenerationSegmentStore(storage as any) + await store.open() + }) + + it('fold → read round-trips deltas and records via sidecar point-reads', async () => { + const meta = await store.fold([gen(1), gen(2), gen(3)]) + expect(meta).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) + expect(meta.checksum).toBeGreaterThan(0) + + expect(store.hasGeneration(2)).toBe(true) + expect(store.hasGeneration(4)).toBe(false) + + const d2 = await store.readDelta(2) + expect(d2?.delta).toEqual({ generation: 2, nouns: [UUID(2)], verbs: [], bytes: 125 }) + expect(d2?.timestamp).toBe(1_700_000_000_002) + + const records = await store.readRecords(3) + expect(records).toHaveLength(2) + expect(records![0]).toEqual({ + kind: 'noun', + id: UUID(300), + record: { metadata: { noun: 'document', v: 3 }, vector: { v: [3, 0] } } + }) + // Point read by id, both kinds. + expect(await store.readRecord(3, 'verb', UUID(301))).toEqual({ + metadata: { noun: 'document', v: 3 }, + vector: { v: [3, 1] } + }) + expect(await store.readRecord(3, 'noun', UUID(999))).toBeNull() + }) + + it('reopen discovers everything from the manifest alone — no listing', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(reopened.segments()).toHaveLength(2) + expect(reopened.hasGeneration(4)).toBe(true) + expect((await reopened.readDelta(1))?.timestamp).toBe(1_700_000_000_001) + }) + + it('a lost sidecar rebuilds from its segment; a damaged segment fails LOUDLY', async () => { + const meta = await store.fold([gen(1), gen(2)]) + const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` + await storage.deleteRawObject(idxPath) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + // Rebuild path: still serves correct data. + expect((await reopened.readRecords(2))!).toHaveLength(2) + + // Now damage the SEGMENT itself: flip a payload byte → CRC mismatch, loud. + const segPath = `${SEGMENTS_PREFIX}/${meta.file}` + const bytes = (await storage.readRawBytes(segPath))! + bytes[bytes.length - 3] ^= 0xff + await storage.writeRawBytes(segPath, bytes) + const damaged = new GenerationSegmentStore(storage as any) + await damaged.open() + ;(damaged as any).sidecars.clear() + await storage.deleteRawObject(idxPath) // force the sequential rebuild over damaged bytes + await expect(damaged.readRecords(2)).rejects.toThrow(/CRC mismatch|damaged/) + }) + + it('D3 reclaim drops whole segments only and bumps compactedBelow', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + await store.fold([gen(5), gen(6)]) + + // Horizon mid-segment-2 (below 4): only segment 1 is FULLY below → drops. + const r1 = await store.dropSegmentsBelow(4) + expect(r1).toEqual({ dropped: 1, compactedBelow: 3 }) + expect(store.hasGeneration(1)).toBe(false) + expect(store.hasGeneration(3)).toBe(true) // partial segment survives whole + + // Bytes actually gone. + expect(await storage.readRawBytes(`${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.bgs`)).toBeNull() + + // Horizon past everything: the rest drop; compactedBelow is durable. + const r2 = await store.dropSegmentsBelow(7) + expect(r2.dropped).toBe(2) + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(reopened.compactedBelow()).toBe(7) + expect(reopened.segments()).toHaveLength(0) + }) + + it('the packed digest is deterministic across reopen and changes with history', async () => { + await store.fold([gen(1), gen(2), gen(3)]) + const atSeal = await store.digestThroughPacked(3) + const midSegment = await store.digestThroughPacked(2) + expect(atSeal).not.toBeNull() + expect(midSegment).not.toBeNull() + expect(midSegment).not.toBe(atSeal) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + expect(await reopened.digestThroughPacked(3)).toBe(atSeal) + expect(await reopened.digestThroughPacked(2)).toBe(midSegment) + + await reopened.fold([gen(4)]) + expect(await reopened.digestThroughPacked(4)).not.toBe(atSeal) + }) + + it('sealed segments are immutable — fold refuses overlap, requires ascending input', async () => { + await store.fold([gen(1), gen(2)]) + await expect(store.fold([gen(2), gen(3)])).rejects.toThrow(/overlaps the packed tier/) + await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) + await expect(store.fold([])).rejects.toThrow(/at least one generation/) + }) +}) From 1201e2554330858df7a419d1c7396aa5395a8c85 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sun, 19 Jul 2026 16:26:10 -0700 Subject: [PATCH 03/10] =?UTF-8?q?feat:=20two-tier=20history=20reads=20+=20?= =?UTF-8?q?the=20repacker=20+=20generationDigest=20=E2=80=94=20D1+D3=20wir?= =?UTF-8?q?ed=20end-to-end?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The packed tier goes live inside GenerationStore: - Two-tier reads: getDelta / readBeforeImage / readGenerationRecords fall through live-tier → sealed segments (live-tier-wins: a crash mid-fold leaves a duplicate representation, never a gap). Cold-open seeds committedRanges from the segment manifest via interval merge — packed generations resolve without their directories existing. - repackHistory({timeBudgetMs, batchGenerations}): folds cold generations (older than the newest 1024) oldest-first into sealed segments, deleting per-generation directories only after segment + manifest are durable. Public API + automatic time-bounded pass at close() (before compaction, so reclaim can drop whole segments); re-representation only — the sole history transform under the archival profile. Early stop = consistent prefix, next pass resumes. - compact(): packed generations reclaim logically in the loop and physically at whole-segment boundaries via dropSegmentsBelow (the frozen partial-segments-wait rule). - generationDigest(g) (D8): deterministic content digest through g — sealed-segment checksum chain + live-tier delta hashes; O(segments + live window); RangeError out of range, GenerationCompactedError below the horizon (a gate can never silently pin reclaimed history). Four end-to-end pins: asOf answers byte-identical across fold + cold reopen with folded dirs physically gone; repack+reclaim composition; digest reopen-stability/divergence/loud-horizon; budget no-op+resume. --- src/brainy.ts | 59 +++++- src/db/generationStore.ts | 217 +++++++++++++++++++- tests/integration/history-repacking.test.ts | 186 +++++++++++++++++ 3 files changed, 454 insertions(+), 8 deletions(-) create mode 100644 tests/integration/history-repacking.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 8ba991dd..9ab3acee 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8265,6 +8265,27 @@ export class Brainy implements BrainyInterface { return this.generationStore.compact(options) } + /** + * @description Repack cold generation history into sealed segments — + * re-representation, never deletion: every record and delta stays readable + * (`asOf()` unchanged); the physical file count drops by orders of + * magnitude. Runs automatically (time-bounded) at `close()`; call this for + * explicit maintenance windows on long-lived writers. The ONLY history + * transform permitted under the archival profile (`retention: 'all'`). + * @param options - `timeBudgetMs` bounds the pass (early stop = consistent + * prefix, next pass resumes); `batchGenerations` sizes each fold. + * @returns Folded generation count and segments created. + */ + async repackHistory(options?: { + timeBudgetMs?: number + batchGenerations?: number + }): Promise<{ foldedGenerations: number; segmentsCreated: number }> { + this.assertWritable('repackHistory') + await this.ensureInitialized() + await this.generationStore.flushPendingSingleOps() + return this.generationStore.repackHistory(options) + } + /** * @description Read-only generational-history footprint for fleet audits: * generation count, total on-disk bytes, generation/timestamp range, the @@ -8292,6 +8313,24 @@ export class Brainy implements BrainyInterface { } } + /** + * @description A deterministic content digest of the generation log through + * `g` (D8 — gate-to-generation provenance): identical history produces the + * identical digest on any machine; divergence produces a different one. + * Release gates and suite verdicts pin `{generation, digest}` and verify + * both at execution time instead of pinning a git commit. O(segments + + * live-tier window), never O(all generations). Throws `RangeError` out of + * range and `GenerationCompactedError` below the horizon — a gate can + * never silently pin reclaimed history. + * @example + * const gate = { generation: brain.generation(), digest: await brain.generationDigest(brain.generation()) } + */ + async generationDigest(g: number): Promise { + await this.ensureInitialized() + await this.generationStore.flushPendingSingleOps() + return this.generationStore.generationDigest(g) + } + /** * @description Drive the adaptive retention byte budget at runtime — the * settable input a machine-level coordinator (e.g. cor's `ResourceManager`, @@ -16192,11 +16231,21 @@ export class Brainy implements BrainyInterface { await this.generationStore.flushPendingSingleOps() } - // Phase 0b: Auto-compact generational history per config.retention (default - // on) BEFORE the generation store closes below. This is THE auto-compaction - // site (8.9.0 — flush() never compacts): time-bounded per pass, respects - // live Db pins and an explicit autoCompact: false; no-op on read-only - // instances. + // Phase 0b: REPACK cold history into sealed segments (D1+D3 — + // re-representation, never deletion; the only history transform under the + // archival profile), then auto-compact per config.retention. Repack runs + // FIRST so bounded-retention reclaim can drop whole segments. Both are + // time-bounded maintenance passes (8.9.0 law: flush() never pays these); + // both are housekeeping — failures warn, never fail a clean shutdown. + if (!this.isReadOnly && this.generationStore) { + try { + await this.generationStore.repackHistory({ timeBudgetMs: 5_000 }) + } catch (error) { + console.warn( + `History repacking failed (non-fatal): ${error instanceof Error ? error.message : String(error)}` + ) + } + } await this.autoCompactHistory() // Phase 1: Flush ALL components in parallel to persist buffered data diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 4e3738d9..aede17a4 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -46,6 +46,8 @@ import type { TxLogEntry } from './types.js' import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js' +import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js' +import { crc32c } from '../utils/crc32c.js' /** * The byte-identical before-images of every id a commit touches, read UNDER @@ -266,6 +268,21 @@ export class GenerationStore { */ private historyBytesTotal: number | null = null + /** + * The packed tier (D1+D3): sealed segments holding folded cold + * generations. Null until {@link open} wires it (and on storage adapters + * without raw-byte primitives — the live tier then carries everything, + * exactly as before the packed tier existed). + */ + private segments: GenerationSegmentStore | null = null + + /** + * Live-tier window: generations newer than `committed - REPACK_LIVE_WINDOW` + * are never folded — the hot tail stays in the per-generation layout the + * write path owns. Matches the resident chain window's scale. + */ + static readonly REPACK_LIVE_WINDOW = 1024 + /** * Model-B per-write group-commit — the in-memory PENDING tier. * @@ -433,6 +450,33 @@ export class GenerationStore { this.factLog = null } + // PACKED TIER (D1+D3): same capability gate as the fact log. Opening + // reads ONE manifest — never a listing of the packed backlog — and seeds + // committedRanges with the sealed ranges so packed generations resolve + // exactly like live ones. + if (storageSupportsFactLog(this.storage)) { + this.segments = new GenerationSegmentStore(this.storage) + await this.segments.open() + const packedRanges = this.segments + .segments() + .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) + .filter(([lo, hi]) => lo <= hi) + if (packedRanges.length > 0) { + // Merge packed (older) + live (newer) interval sets — both ascending; + // coalesce adjacency so range arithmetic stays interval-exact. + const merged: Array<[number, number]> = [] + for (const r of [...packedRanges, ...this.committedRanges].sort((a, b) => a[0] - b[0])) { + const last = merged[merged.length - 1] + if (last && r[0] <= last[1] + 1) last[1] = Math.max(last[1], r[1]) + else merged.push([r[0], r[1]]) + } + this.committedRanges = merged + } + this.horizonGen = Math.max(this.horizonGen, this.segments.compactedBelow() - 1) + } else { + this.segments = null + } + // Hook single-op write batches so generation() is always meaningful. // Suppressed while a transact batch executes (the batch is ONE generation). if (!options?.readOnly) { @@ -500,6 +544,51 @@ export class GenerationStore { * deltas (cache-bounded reads). * @returns Counts, bytes, generation range, and the compaction horizon. */ + /** + * @description D8 (gate-to-generation provenance): a deterministic content + * digest of the generation log THROUGH `g` — identical history ⇒ identical + * digest on any machine; any divergence (different records, different + * order, reclaimed range) ⇒ different digest. Composed from the packed + * tier's sealed-segment checksum chain (O(segments)) plus the live tier's + * per-generation delta digests (O(live window at most)). Release gates pin + * {generation, digest} and verify both at execution time. + * @param g - The generation to digest through (≤ committed). + * @returns A hex digest string, stable across reopen and repacking states + * ONLY for fully-packed prefixes — repacking changes representation, so + * the composed digest is defined over CONTENT: live-tier gens hash their + * delta + record ids, packed gens hash via frame CRCs. A gate should pin + * after a repack pass for long-term stability, or re-pin on repack. + */ + async generationDigest(g: number): Promise { + if (!Number.isInteger(g) || g < 1 || g > this.committed) { + throw new RangeError( + `generationDigest(): generation ${g} is out of range [1, ${this.committed}]` + ) + } + if (g <= this.horizonGen) { + throw new GenerationCompactedError(g, this.horizonGen) + } + let digest = 0 + const enc = new TextEncoder() + if (this.segments) { + const packed = await this.segments.digestThroughPacked(g) + if (packed !== null) digest = packed + } + // Live-tier composition: every committed gen ≤ g not covered by a sealed + // segment hashes its delta content in ascending order. + for (const gen of this.committedGensAsc()) { + if (gen > g) break + if (this.segments?.hasGeneration(gen)) continue + const delta = await this.getDelta(gen) + digest = crc32c( + enc.encode( + `${digest}:${gen}:${delta.timestamp}:${[...delta.nouns].sort().join(',')}:${[...delta.verbs].sort().join(',')}` + ) + ) + } + return digest.toString(16).padStart(8, '0') + } + async historyStats(): Promise<{ generations: number bytes: number @@ -538,14 +627,17 @@ export class GenerationStore { try { paths = await this.storage.listRawObjects(`${GENERATIONS_PREFIX}/${gen}/prev`) } catch { - return [] + paths = [] } const records: GenerationRecord[] = [] for (const p of paths) { const record = (await this.storage.readRawObject(p)) as GenerationRecord | null if (record) records.push(record) } - return records + if (records.length > 0) return records + // Two-tier: folded generations serve their record-set from the segment. + const packed = await this.segments?.readRecords(gen) + return packed ? (packed.map((r) => r.record) as GenerationRecord[]) : [] } /** @@ -1783,9 +1875,15 @@ export class GenerationStore { if (pending) { return (kind === 'noun' ? pending.nouns : pending.verbs).get(id) ?? null } - return (await this.storage.readRawObject( + const live = (await this.storage.readRawObject( `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json` )) as GenerationRecord | null + if (live) return live + // Two-tier: the packed tier serves folded generations (live-tier-wins). + if (this.segments?.hasGeneration(gen)) { + return (await this.segments.readRecord(gen, kind, id)) as GenerationRecord | null + } + return null } /** @@ -2132,6 +2230,21 @@ export class GenerationStore { `${GENERATIONS_PREFIX}/${gen}/tx.json` )) as GenerationDelta | null if (delta === null) { + // Two-tier read (D1+D3): not in the live tier → the packed tier. + // Live-tier-wins ordering (a crash mid-fold leaves a duplicate, never + // a gap), so the segment lookup runs only after the live miss. + const packed = await this.segments?.readDelta(gen) + if (packed) { + const d = packed.delta as GenerationDelta + const entry = { + nouns: new Set(d.nouns), + verbs: new Set(d.verbs), + timestamp: packed.timestamp, + bytes: d.bytes ?? 0 + } + this.setDelta(gen, entry) + return entry + } throw new Error( `Generation delta missing: ${GENERATIONS_PREFIX}/${gen}/tx.json ` + `(store corrupted or records removed outside compactHistory())` @@ -2213,6 +2326,94 @@ export class GenerationStore { * @param options - Retention caps (see {@link CompactHistoryOptions}). * @returns Count of removed record-sets and the new horizon. */ + /** + * @description The REPACKER (D1+D3+repacking): fold cold live-tier + * generations into sealed segments — re-representation, never deletion. + * Every record and delta stays readable (asOf/chains unchanged); the + * per-generation directories are deleted only AFTER their segment is + * durable (crash between = duplicate representation, resolved + * live-tier-wins by every reader; never a gap). This is the transform that + * takes a 70k-file history to tens of segment files, and the ONLY history + * transform permitted under the archival profile. + * + * Folds oldest-first, contiguous from the packed boundary, in batches, and + * stops at the live window ({@link GenerationStore.REPACK_LIVE_WINDOW}) + * or when `timeBudgetMs` is spent — an early stop is a consistent prefix; + * the next pass resumes. + */ + async repackHistory(options?: { timeBudgetMs?: number; batchGenerations?: number }): Promise<{ + foldedGenerations: number + segmentsCreated: number + }> { + if (!this.segments) return { foldedGenerations: 0, segmentsCreated: 0 } + const segments = this.segments + return this.withMutex(async () => { + const deadline = + options?.timeBudgetMs !== undefined ? Date.now() + options.timeBudgetMs : undefined + const batchSize = options?.batchGenerations ?? 512 + const coldCeiling = this.committed - GenerationStore.REPACK_LIVE_WINDOW + const packedThrough = + segments.segments().length > 0 + ? segments.segments()[segments.segments().length - 1].lastGeneration + : 0 + + // Cold, unpacked, committed generations — ascending, contiguous scan. + const eligible: number[] = [] + for (const gen of this.committedGensAsc()) { + if (gen > coldCeiling) break + if (gen <= packedThrough) continue // already packed (dup fold barred) + if (this.pendingBuffer.has(gen)) continue // un-flushed = live by definition + eligible.push(gen) + } + + let folded = 0 + let segmentsCreated = 0 + for (let i = 0; i < eligible.length; i += batchSize) { + if (deadline !== undefined && Date.now() >= deadline) break + const batch = eligible.slice(i, i + batchSize) + const foldInput: FoldGeneration[] = [] + for (const gen of batch) { + const delta = (await this.storage.readRawObject( + `${GENERATIONS_PREFIX}/${gen}/tx.json` + )) as GenerationDelta | null + if (delta === null) { + // Already folded by a prior crashed pass whose dirs were removed, + // or damage — getDelta's two-tier read decides which, loudly, + // when someone asks. Skip; never fold a generation we cannot read. + continue + } + const records: FoldGeneration['records'] = [] + for (const [kind, ids] of [ + ['noun', delta.nouns] as const, + ['verb', delta.verbs] as const + ]) { + for (const id of ids) { + const record = await this.storage.readRawObject( + `${GENERATIONS_PREFIX}/${gen}/prev/${id}.json` + ) + if (record) records.push({ kind, id, record }) + } + } + foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) + } + if (foldInput.length === 0) continue + await segments.fold(foldInput) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of foldInput) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + } + folded += foldInput.length + } + if (folded > 0) { + prodLog.info( + `[GenerationStore] repacked ${folded} cold generation(s) into ${segmentsCreated} segment(s) — history preserved, file count reduced` + ) + } + return { foldedGenerations: folded, segmentsCreated } + }) + } + async compact(options?: CompactHistoryOptions): Promise { return this.withMutex(async () => { const minPinned = this.minPinnedGeneration() @@ -2304,6 +2505,16 @@ export class GenerationStore { // Reclaimed generations leave the per-id chains stale → rebuild on next read. this.invalidateChains() this.horizonGen = Math.max(this.horizonGen, highestRemoved) + // Packed-tier reclaim (D3): a packed generation's bytes live in a + // sealed segment — removeRawPrefix above was a no-op for it. Drop + // WHOLE segments now fully below the horizon; a partially-reclaimed + // segment keeps its bytes until the boundary passes it (the frozen + // partial-segments-wait rule; logical reclamation above still holds — + // the generations left committedRanges and asOf below the horizon + // throws regardless). + if (this.segments) { + await this.segments.dropSegmentsBelow(this.horizonGen + 1) + } const manifest: GenerationManifest = { version: 1, generation: this.committed, diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts new file mode 100644 index 00000000..2bcee038 --- /dev/null +++ b/tests/integration/history-repacking.test.ts @@ -0,0 +1,186 @@ +/** + * @module tests/integration/history-repacking + * @description The D1+D3 two-tier history lifecycle end-to-end on a real + * brain. Laws: (1) repacking is RE-REPRESENTATION — after folding, every + * asOf() read below the fold boundary answers exactly as before, across a + * cold reopen; (2) folded per-generation directories are physically gone + * (the file-count cure is real, not cosmetic); (3) repack + reclaim compose: + * bounded retention after repacking drops whole segments and asOf below the + * horizon throws GenerationCompactedError; (4) repackHistory is explicit + * API and time-bounded (spent budget = consistent no-op). + * + * Uses a tiny REPACK_LIVE_WINDOW override so a small history has a cold + * tier at all (the production window is 1024). + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as path from 'node:path' +import * as os from 'node:os' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { GenerationStore } from '../../src/db/generationStore.js' +import { GenerationCompactedError } from '../../src/db/errors.js' +import { SEGMENTS_PREFIX } from '../../src/db/generationSegments.js' + +const stub = async (text: string): Promise => { + const h = text.split('').reduce((a, c) => a + c.charCodeAt(0), 0) + return new Array(384).fill(0).map((_, i) => Math.sin(h + i)) +} + +const openBrain = async (dir: string): Promise => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + embeddingFunction: stub + }) + await brain.init() + return brain +} + +describe('history repacking — the two-tier lifecycle', () => { + const dirs: string[] = [] + const tempDir = (): string => { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-repack-')) + dirs.push(d) + return d + } + const originalWindow = GenerationStore.REPACK_LIVE_WINDOW + + afterEach(() => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = originalWindow + for (const d of dirs.splice(0)) { + try { + fs.rmSync(d, { recursive: true, force: true }) + } catch { + /* best effort */ + } + } + }) + + it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 + const dir = tempDir() + const brain = await openBrain(dir) + + const id = await brain.add({ + data: 'versioned-entity', + type: NounType.Document, + metadata: { v: 0 } + }) + for (let v = 1; v <= 10; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + + // Ground truth BEFORE repacking: capture asOf views for early generations. + const before: Record = {} + for (const g of [2, 4, 6]) { + const db = await brain.asOf(g) + before[g] = (await db.get(id))?.metadata?.v as number + await db.release() + } + + const result = await brain.repackHistory() + expect(result.foldedGenerations).toBeGreaterThan(0) + expect(result.segmentsCreated).toBeGreaterThan(0) + + // The folded per-generation directories are PHYSICALLY gone… + const genDirs = fs + .readdirSync(path.join(dir, '_generations'), { withFileTypes: true }) + .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)).length + expect(genDirs).toBeLessThanOrEqual(4) // live window (3) + at most the newest + // …and the segment tier exists (the filesystem adapter stores objects + // gzipped, so the manifest may live at either spelling). + const segDir = path.join(dir, SEGMENTS_PREFIX) + expect( + fs.existsSync(path.join(segDir, 'manifest.json')) || + fs.existsSync(path.join(segDir, 'manifest.json.gz')) + ).toBe(true) + expect(fs.readdirSync(segDir).some((f) => f.endsWith('.bgs'))).toBe(true) + + // Same asOf answers from the packed tier, same process… + for (const g of [2, 4, 6]) { + const db = await brain.asOf(g) + expect((await db.get(id))?.metadata?.v).toBe(before[g]) + await db.release() + } + await brain.close() + + // …and across a COLD REOPEN (manifest discovery, no live dirs to list). + const reopened = await openBrain(dir) + for (const g of [2, 4, 6]) { + const db = await reopened.asOf(g) + expect((await db.get(id))?.metadata?.v).toBe(before[g]) + await db.release() + } + expect((await reopened.get(id))?.metadata?.v).toBe(10) // live state untouched + await reopened.close() + }) + + it('repack + bounded reclaim compose: whole segments drop, horizon is loud', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'reclaim-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 8; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + await brain.repackHistory() + + // Reclaim down to the 3 newest generations — packed segments below the + // horizon drop whole; asOf below throws loudly. + const res = await brain.compactHistory({ maxGenerations: 3 }) + expect(res.removedGenerations).toBeGreaterThan(0) + await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) + expect((await brain.get(id))?.metadata?.v).toBe(8) + await brain.close() + }) + + it('generationDigest: reopen-stable, divergence-sensitive, loud below the horizon', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'digest-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + await brain.repackHistory() + + const gen = brain.generation() + const atHead = await brain.generationDigest(gen) + const atMid = await brain.generationDigest(3) + expect(atHead).toMatch(/^[0-9a-f]{8}$/) + expect(atMid).not.toBe(atHead) // more history ⇒ different digest + await brain.close() + + // Reopen-stable: same history, same digests (packed prefix stability). + const reopened = await openBrain(dir) + expect(await reopened.generationDigest(gen)).toBe(atHead) + expect(await reopened.generationDigest(3)).toBe(atMid) + + // New history diverges the head digest. + await reopened.update({ id, metadata: { v: 7 } }) + await reopened.flush() + expect(await reopened.generationDigest(reopened.generation())).not.toBe(atHead) + + // Below the horizon: LOUD, never a silent pin of reclaimed history. + await reopened.compactHistory({ maxGenerations: 2 }) + await expect(reopened.generationDigest(1)).rejects.toBeInstanceOf(GenerationCompactedError) + await reopened.close() + }) + + it('a spent time budget is a consistent no-op; the next pass resumes', async () => { + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2 + const dir = tempDir() + const brain = await openBrain(dir) + const id = await brain.add({ data: 'budget-probe', type: NounType.Document, metadata: { v: 0 } }) + for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } }) + await brain.flush() + + const bounded = await brain.repackHistory({ timeBudgetMs: 0 }) + expect(bounded).toEqual({ foldedGenerations: 0, segmentsCreated: 0 }) + + const resumed = await brain.repackHistory() + expect(resumed.foldedGenerations).toBeGreaterThan(0) + const db = await brain.asOf(3) + expect((await db.get(id))?.metadata?.v).toBeDefined() + await db.release() + await brain.close() + }) +}) From 9a5a9cccbcab8d67e475df13458e5c9a4081e9a9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 22 Jul 2026 16:31:45 +0200 Subject: [PATCH 04/10] ci: run the pipeline on the forge --- .forgejo/workflows/ci.yml | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .forgejo/workflows/ci.yml diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 00000000..cdb2ab14 --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + pull_request: + +jobs: + node: + name: Node ${{ matrix.node-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: ['22', '24'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + - run: npm ci + - run: npm run test:unit + + bun: + name: Bun (latest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + - uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + - run: npm ci + # test:bun imports the built dist/, so build first. + - run: npm run build + # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). + - run: npm run test:bun From 415e824a1da44a1109f54818721289f5e9a42af2 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 23 Jul 2026 11:09:43 -0700 Subject: [PATCH 05/10] =?UTF-8?q?chore:=20the=20forge=20is=20the=20address?= =?UTF-8?q?=20=E2=80=94=20retire=20the=20archived=20mirror=20from=20every?= =?UTF-8?q?=20live=20surface?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruled today: the project's one public home is source.soulcraft.com. The old public repo is archived history and no longer part of any release. - package.json repository/homepage/bugs now point at the forge (this is what the npm page links as Repository/Homepage/Issues) - README CI badge reads the forge pipeline; CONTRIBUTING drops the mirror paragraph (forge account or email patch were already the ruled contribution paths) - release.sh: mirror push + external release step removed; publishes go forge-first (box-held write token, temp userconfig so the token never hits argv; a forge-publish failure aborts before the storefront so the pair can never diverge), then npmjs with the scope-override pin (the fleet npmrc maps @soulcraft to the forge and scope mappings beat --registry); release page created via forge API when a token is present, loud skip otherwise; changelog compare links point home - dead external CI workflow removed (.forgejo/workflows/ci.yml is the live pipeline) Historical CHANGELOG links to the archive stay as written - history is history and the archive serves them read-only. --- .github/workflows/ci.yml | 40 ---------------------- CONTRIBUTING.md | 4 --- README.md | 2 +- package.json | 6 ++-- scripts/release.sh | 71 +++++++++++++++++++++++++--------------- 5 files changed, 48 insertions(+), 75 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index cdb2ab14..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: CI - -on: - push: - pull_request: - -jobs: - node: - name: Node ${{ matrix.node-version }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - node-version: ['22', '24'] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: npm - - run: npm ci - - run: npm run test:unit - - bun: - name: Bun (latest) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - run: npm ci - # test:bun imports the built dist/, so build first. - - run: npm run build - # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). - - run: npm run test:bun diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef9c4a51..d277091d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,10 +10,6 @@ The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/bra It's anonymously readable and cloneable — no account needed to browse, clone, or build. -**github.com/soulcraftlabs/brainy** is a public read-only mirror. It's a fine -place to read code or star the project, but issues and pull requests opened -there won't be picked up — please use one of the paths below instead. - ## How to contribute **Found a bug, or have an idea?** Email **brainy@soulcraft.com**. No account, diff --git a/README.md b/README.md index 2fc42060..2caf6493 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

npm version npm downloads - CI + CI Documentation MIT License TypeScript diff --git a/package.json b/package.json index e4bc8144..a3ece83c 100644 --- a/package.json +++ b/package.json @@ -128,13 +128,13 @@ "publishConfig": { "access": "public" }, - "homepage": "https://github.com/soulcraftlabs/brainy", + "homepage": "https://source.soulcraft.com/soulcraft/brainy", "bugs": { - "url": "https://github.com/soulcraftlabs/brainy/issues" + "url": "https://source.soulcraft.com/soulcraft/brainy/issues" }, "repository": { "type": "git", - "url": "git+https://github.com/soulcraftlabs/brainy.git" + "url": "git+https://source.soulcraft.com/soulcraft/brainy.git" }, "files": [ "dist/**/*.js", diff --git a/scripts/release.sh b/scripts/release.sh index 42f5b345..43fa50bd 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -142,7 +142,7 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://github.com/soulcraftlabs/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) ${COMMITS} " @@ -175,42 +175,59 @@ echo -e "${BLUE}7️⃣ Creating git tag v${NEW_VERSION}...${NC}" git tag -a "v${NEW_VERSION}" -m "Release v${NEW_VERSION}" echo -e "${GREEN}✅ Tag created${NC}\n" -# Step 9: Push to origin (source of truth) and the public GitHub mirror +# Step 9: Push to origin — the forge is the one home (ruled 2026-07-23; the +# old public GitHub repo is archived history, no longer part of any release). echo -e "${BLUE}8️⃣ Pushing to origin...${NC}" git push --follow-tags origin "$CURRENT_BRANCH" echo -e "${GREEN}✅ Pushed to origin${NC}\n" -# The public GitHub repo is a mirror of origin with an unknown sync cadence. -# `gh release create` below targets GitHub directly: if the new tag hasn't -# reached GitHub yet, gh would CREATE it — pointed at GitHub's default-branch -# head, i.e. the wrong commit. Push branch+tag to GitHub explicitly, then -# verify the tag resolves there to the same commit before any release is cut. -GITHUB_URL="https://github.com/soulcraftlabs/brainy.git" -echo -e "${BLUE}8️⃣½ Pushing to the public GitHub mirror...${NC}" -git push --follow-tags "$GITHUB_URL" "$CURRENT_BRANCH" -LOCAL_TAG_SHA="$(git rev-parse "v${NEW_VERSION}^{}")" -GITHUB_TAG_SHA="$(git ls-remote --tags "$GITHUB_URL" "v${NEW_VERSION}^{}" | cut -f1)" -if [ "$LOCAL_TAG_SHA" != "$GITHUB_TAG_SHA" ]; then - echo -e "${RED}❌ Tag v${NEW_VERSION} on GitHub (${GITHUB_TAG_SHA:-absent}) does not match local (${LOCAL_TAG_SHA}) — aborting before npm publish. Fix the mirror, then re-run.${NC}" +# Step 10: Publish — forge FIRST (home), npmjs second (the world's storefront). +# The fleet-wide ~/.npmrc maps the @soulcraft scope to the forge registry, and +# a scope mapping BEATS `--registry` on the command line — so each publish +# names its registry via the scope override explicitly. Nothing implicit. +FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" +FORGE_NPM_TOKEN_FILE="$HOME/.config/soulcraft/npm-publish-brainy.token" +echo -e "${BLUE}9️⃣ Publishing to the forge registry (home)...${NC}" +if [ -f "$FORGE_NPM_TOKEN_FILE" ]; then + TMPRC="$(mktemp)" + chmod 600 "$TMPRC" + { + echo "@soulcraft:registry=${FORGE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=$(cat "$FORGE_NPM_TOKEN_FILE")" + } > "$TMPRC" + if npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then + echo -e "${GREEN}✅ Published to the forge${NC}\n" + else + rm -f "$TMPRC" + echo -e "${RED}❌ Forge publish FAILED — aborting before npmjs so the pair never diverges. Fix and re-run.${NC}" + exit 1 + fi + rm -f "$TMPRC" +else + echo -e "${RED}❌ Forge publish token missing (${FORGE_NPM_TOKEN_FILE}) — aborting. The forge is home; publish it first or restage the token.${NC}" exit 1 fi -echo -e "${GREEN}✅ GitHub mirror has the tag at the right commit${NC}\n" -# Step 10: Publish to npm -echo -e "${BLUE}9️⃣ Publishing to npm (dist-tag: ${NPM_TAG})...${NC}" -npm publish --tag "$NPM_TAG" +echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}" +npm publish --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" # Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. -npm access get status @soulcraft/brainy || true -echo -e "${GREEN}✅ Published to npm${NC}\n" +npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true +echo -e "${GREEN}✅ Published to npmjs${NC}\n" -# Step 11: Create GitHub release -echo -e "${BLUE}🔟 Creating GitHub release...${NC}" -if [ "$PRERELEASE" = true ]; then - gh release create "v${NEW_VERSION}" --generate-notes --prerelease +# Step 11: Release object on the forge (presentational — the tag, CHANGELOG, +# and RELEASES.md are the record; this just gives the forge UI a release page). +echo -e "${BLUE}🔟 Creating forge release...${NC}" +if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then + if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ + -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ + -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then + echo -e "${GREEN}✅ Forge release created${NC}\n" + else + echo -e "${RED}⚠️ Forge release API call failed — tag + CHANGELOG remain the record; create the release page via the forge UI if wanted${NC}\n" + fi else - gh release create "v${NEW_VERSION}" --generate-notes + echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" fi -echo -e "${GREEN}✅ GitHub release created${NC}\n" # Step 12: Push public docs to the soulcraft.com docs ingest door # (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when @@ -229,4 +246,4 @@ echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}" -echo -e "🐙 GitHub: ${BLUE}https://github.com/soulcraftlabs/brainy/releases/tag/v${NEW_VERSION}${NC}" +echo -e "🏠 Forge: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" From 22702b81c0557df6e0f10713f958beb956d06044 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 23 Jul 2026 11:09:43 -0700 Subject: [PATCH 06/10] =?UTF-8?q?chore:=20the=20forge=20is=20the=20address?= =?UTF-8?q?=20=E2=80=94=20retire=20the=20archived=20mirror=20from=20every?= =?UTF-8?q?=20live=20surface?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruled today: the project's one public home is source.soulcraft.com. The old public repo is archived history and no longer part of any release. - package.json repository/homepage/bugs now point at the forge (this is what the npm page links as Repository/Homepage/Issues) - README CI badge reads the forge pipeline; CONTRIBUTING drops the mirror paragraph (forge account or email patch were already the ruled contribution paths) - release.sh: mirror push + external release step removed; publishes go forge-first (box-held write token, temp userconfig so the token never hits argv; a forge-publish failure aborts before the storefront so the pair can never diverge), then npmjs with the scope-override pin (the fleet npmrc maps @soulcraft to the forge and scope mappings beat --registry); release page created via forge API when a token is present, loud skip otherwise; changelog compare links point home - dead external CI workflow removed (.forgejo/workflows/ci.yml is the live pipeline) Historical CHANGELOG links to the archive stay as written - history is history and the archive serves them read-only. --- .github/workflows/ci.yml | 40 ---------------------- CONTRIBUTING.md | 4 --- README.md | 2 +- package.json | 6 ++-- scripts/release.sh | 71 +++++++++++++++++++++++++--------------- 5 files changed, 48 insertions(+), 75 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index cdb2ab14..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: CI - -on: - push: - pull_request: - -jobs: - node: - name: Node ${{ matrix.node-version }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - node-version: ['22', '24'] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: npm - - run: npm ci - - run: npm run test:unit - - bun: - name: Bun (latest) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: npm - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - run: npm ci - # test:bun imports the built dist/, so build first. - - run: npm run build - # Bun as a runtime is the supported Bun story (`bun add` / `bun run`). - - run: npm run test:bun diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef9c4a51..d277091d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,10 +10,6 @@ The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/bra It's anonymously readable and cloneable — no account needed to browse, clone, or build. -**github.com/soulcraftlabs/brainy** is a public read-only mirror. It's a fine -place to read code or star the project, but issues and pull requests opened -there won't be picked up — please use one of the paths below instead. - ## How to contribute **Found a bug, or have an idea?** Email **brainy@soulcraft.com**. No account, diff --git a/README.md b/README.md index 2fc42060..2caf6493 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

npm version npm downloads - CI + CI Documentation MIT License TypeScript diff --git a/package.json b/package.json index e4bc8144..a3ece83c 100644 --- a/package.json +++ b/package.json @@ -128,13 +128,13 @@ "publishConfig": { "access": "public" }, - "homepage": "https://github.com/soulcraftlabs/brainy", + "homepage": "https://source.soulcraft.com/soulcraft/brainy", "bugs": { - "url": "https://github.com/soulcraftlabs/brainy/issues" + "url": "https://source.soulcraft.com/soulcraft/brainy/issues" }, "repository": { "type": "git", - "url": "git+https://github.com/soulcraftlabs/brainy.git" + "url": "git+https://source.soulcraft.com/soulcraft/brainy.git" }, "files": [ "dist/**/*.js", diff --git a/scripts/release.sh b/scripts/release.sh index 42f5b345..43fa50bd 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -142,7 +142,7 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://github.com/soulcraftlabs/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) ${COMMITS} " @@ -175,42 +175,59 @@ echo -e "${BLUE}7️⃣ Creating git tag v${NEW_VERSION}...${NC}" git tag -a "v${NEW_VERSION}" -m "Release v${NEW_VERSION}" echo -e "${GREEN}✅ Tag created${NC}\n" -# Step 9: Push to origin (source of truth) and the public GitHub mirror +# Step 9: Push to origin — the forge is the one home (ruled 2026-07-23; the +# old public GitHub repo is archived history, no longer part of any release). echo -e "${BLUE}8️⃣ Pushing to origin...${NC}" git push --follow-tags origin "$CURRENT_BRANCH" echo -e "${GREEN}✅ Pushed to origin${NC}\n" -# The public GitHub repo is a mirror of origin with an unknown sync cadence. -# `gh release create` below targets GitHub directly: if the new tag hasn't -# reached GitHub yet, gh would CREATE it — pointed at GitHub's default-branch -# head, i.e. the wrong commit. Push branch+tag to GitHub explicitly, then -# verify the tag resolves there to the same commit before any release is cut. -GITHUB_URL="https://github.com/soulcraftlabs/brainy.git" -echo -e "${BLUE}8️⃣½ Pushing to the public GitHub mirror...${NC}" -git push --follow-tags "$GITHUB_URL" "$CURRENT_BRANCH" -LOCAL_TAG_SHA="$(git rev-parse "v${NEW_VERSION}^{}")" -GITHUB_TAG_SHA="$(git ls-remote --tags "$GITHUB_URL" "v${NEW_VERSION}^{}" | cut -f1)" -if [ "$LOCAL_TAG_SHA" != "$GITHUB_TAG_SHA" ]; then - echo -e "${RED}❌ Tag v${NEW_VERSION} on GitHub (${GITHUB_TAG_SHA:-absent}) does not match local (${LOCAL_TAG_SHA}) — aborting before npm publish. Fix the mirror, then re-run.${NC}" +# Step 10: Publish — forge FIRST (home), npmjs second (the world's storefront). +# The fleet-wide ~/.npmrc maps the @soulcraft scope to the forge registry, and +# a scope mapping BEATS `--registry` on the command line — so each publish +# names its registry via the scope override explicitly. Nothing implicit. +FORGE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" +FORGE_NPM_TOKEN_FILE="$HOME/.config/soulcraft/npm-publish-brainy.token" +echo -e "${BLUE}9️⃣ Publishing to the forge registry (home)...${NC}" +if [ -f "$FORGE_NPM_TOKEN_FILE" ]; then + TMPRC="$(mktemp)" + chmod 600 "$TMPRC" + { + echo "@soulcraft:registry=${FORGE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=$(cat "$FORGE_NPM_TOKEN_FILE")" + } > "$TMPRC" + if npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then + echo -e "${GREEN}✅ Published to the forge${NC}\n" + else + rm -f "$TMPRC" + echo -e "${RED}❌ Forge publish FAILED — aborting before npmjs so the pair never diverges. Fix and re-run.${NC}" + exit 1 + fi + rm -f "$TMPRC" +else + echo -e "${RED}❌ Forge publish token missing (${FORGE_NPM_TOKEN_FILE}) — aborting. The forge is home; publish it first or restage the token.${NC}" exit 1 fi -echo -e "${GREEN}✅ GitHub mirror has the tag at the right commit${NC}\n" -# Step 10: Publish to npm -echo -e "${BLUE}9️⃣ Publishing to npm (dist-tag: ${NPM_TAG})...${NC}" -npm publish --tag "$NPM_TAG" +echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}" +npm publish --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" # Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. -npm access get status @soulcraft/brainy || true -echo -e "${GREEN}✅ Published to npm${NC}\n" +npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true +echo -e "${GREEN}✅ Published to npmjs${NC}\n" -# Step 11: Create GitHub release -echo -e "${BLUE}🔟 Creating GitHub release...${NC}" -if [ "$PRERELEASE" = true ]; then - gh release create "v${NEW_VERSION}" --generate-notes --prerelease +# Step 11: Release object on the forge (presentational — the tag, CHANGELOG, +# and RELEASES.md are the record; this just gives the forge UI a release page). +echo -e "${BLUE}🔟 Creating forge release...${NC}" +if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then + if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ + -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ + -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then + echo -e "${GREEN}✅ Forge release created${NC}\n" + else + echo -e "${RED}⚠️ Forge release API call failed — tag + CHANGELOG remain the record; create the release page via the forge UI if wanted${NC}\n" + fi else - gh release create "v${NEW_VERSION}" --generate-notes + echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" fi -echo -e "${GREEN}✅ GitHub release created${NC}\n" # Step 12: Push public docs to the soulcraft.com docs ingest door # (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when @@ -229,4 +246,4 @@ echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}" -echo -e "🐙 GitHub: ${BLUE}https://github.com/soulcraftlabs/brainy/releases/tag/v${NEW_VERSION}${NC}" +echo -e "🏠 Forge: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" From 003e2a74ea7dd2598022b2ca6ee3784d85214662 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 24 Jul 2026 16:01:41 -0700 Subject: [PATCH 07/10] fix: transaction timeouts are a typed no-hot-retry contract; engine-side non-retry pinned; dead transaction path removed A production incident: a native-provider op ground 38-40s inside a transaction, blew the apply-phase budget, rolled back, and a downstream pipeline hot-retried the identical operation into a 6-minute CPU storm. Brainy itself never auto-retried the timeout; the gap was that TransactionTimeoutError only said "retryable" in prose, with nothing machine-readable for a caller to branch on. - TransactionTimeoutError gains two typed, always-true fields: retryable (a later attempt may succeed once the slowness resolves or the budget is raised) and hotRetryUnsafe (an immediate identical retry re-pays the full cost that just timed out and can cascade into a CPU storm -- callers must latch and back off, never loop). context's existing telemetry fields (timeoutMs, operationIndex, elapsedMs, totalOperations, operationName) are now documented as the caller's backoff inputs. - Updated the "retryable" doc-prose sites (transact()'s timeoutMs option, transactionBudgetFloorMs, Transaction.execute()'s contract) to point at the new fields instead of bare prose. - Regression pin (tests/unit/transaction/timeout-never-internally-retried.test.ts): an execution counter proves the engine never re-drives a timed-out operation, through both the single-op engine TransactionManager/Transaction drives for every single-record write, and add()'s upsert-race retry loop (which must exit on the first TransactionTimeoutError, never treat it like the lost-insert-race signal it retries on). - Removed TransactionManager.executeTransactionWithResult -- zero callers anywhere in the codebase. --- src/db/types.ts | 10 +- src/transaction/Transaction.ts | 7 +- src/transaction/TransactionManager.ts | 29 ---- src/transaction/errors.ts | 45 +++++- src/types/brainy.types.ts | 6 +- .../TransactionManager.unit.test.ts | 37 ----- .../timeout-never-internally-retried.test.ts | 141 ++++++++++++++++++ 7 files changed, 199 insertions(+), 76 deletions(-) create mode 100644 tests/unit/transaction/timeout-never-internally-retried.test.ts diff --git a/src/db/types.ts b/src/db/types.ts index 4c8a4957..a7f86a89 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -121,9 +121,13 @@ export interface TransactOptions { * with the batch: `max(30 000, opCount × 2 000)` — production imports on * network-attached disks measure ~2 s per operation, so a flat 30 s budget * silently capped honest bulk work at ~15 operations. A tripped budget - * rolls the whole batch back and throws a retryable - * `TransactionTimeoutError` naming the operation it stopped at, the batch - * size, and the elapsed/budget times. + * rolls the whole batch back and throws a `TransactionTimeoutError` naming + * the operation it stopped at, the batch size, and the elapsed/budget + * times. That error is retryable-with-latch, never hot-retry: its + * `retryable` field says a later attempt may succeed, its + * `hotRetryUnsafe` field says an immediate identical retry re-pays the + * full cost that just timed out — callers must latch and back off, never + * loop. */ timeoutMs?: number } diff --git a/src/transaction/Transaction.ts b/src/transaction/Transaction.ts index 79b53006..ede65e83 100644 --- a/src/transaction/Transaction.ts +++ b/src/transaction/Transaction.ts @@ -62,9 +62,10 @@ const DEFAULT_BUDGET_FLOOR_MS = 30_000 * NEXT operation may start (see {@link Transaction.execute}), never whether * already-completed work is rolled back after the fact. A trip mid-batch * still rolls back every operation applied so far, atomically, and throws a - * retryable, fully-labeled TransactionTimeoutError — that zero-loss guarantee - * doesn't change; only the point at which the clock stops mattering does (at - * the last operation, not one check later). + * fully-labeled `TransactionTimeoutError` — retryable-with-latch, never + * hot-retry (see its `retryable` and `hotRetryUnsafe` fields) — that + * zero-loss guarantee doesn't change; only the point at which the clock + * stops mattering does (at the last operation, not one check later). * * @param opCount - Number of operations in the batch. * @param override - A full override for this call; wins over everything else. diff --git a/src/transaction/TransactionManager.ts b/src/transaction/TransactionManager.ts index 0f13b6a3..5abf48ba 100644 --- a/src/transaction/TransactionManager.ts +++ b/src/transaction/TransactionManager.ts @@ -19,7 +19,6 @@ import { Transaction } from './Transaction.js' import { TransactionFunction, - TransactionResult, TransactionOptions } from './types.js' import { TransactionError } from './errors.js' @@ -105,34 +104,6 @@ export class TransactionManager { } } - /** - * Execute a transaction and return detailed result - */ - async executeTransactionWithResult( - fn: TransactionFunction, - options?: TransactionOptions - ): Promise> { - const startTime = Date.now() - const transaction = new Transaction(options) - - try { - const value = await fn(transaction) - await transaction.execute() - - const executionTimeMs = Date.now() - startTime - - return { - value, - operationCount: transaction.getOperationCount(), - executionTimeMs - } - - } catch (error) { - // Transaction failed - throw error - } - } - /** * Get transaction statistics */ diff --git a/src/transaction/errors.ts b/src/transaction/errors.ts index c270d0ed..d8382a4b 100644 --- a/src/transaction/errors.ts +++ b/src/transaction/errors.ts @@ -73,14 +73,47 @@ export class InvalidTransactionStateError extends TransactionError { /** * Error for transaction timeout + * + * Machine-readable no-hot-retry contract: {@link retryable} and + * {@link hotRetryUnsafe} are both always `true` on this class — they exist + * so a caller can branch on the *shape* of the error instead of parsing + * message text. Read them together: the operation may eventually succeed, + * but never by looping on it immediately. + * + * `context` (inherited from {@link TransactionError}) carries the caller's + * backoff inputs — see the field docs below. */ export class TransactionTimeoutError extends TransactionError { + /** + * The failed operation MAY succeed on a later attempt — once the + * underlying slowness resolves (e.g. a cold page cache warms up) or the + * budget is deliberately raised (`transactionBudgetFloorMs`, or a larger + * `timeoutMs` override on the batch). This is a statement about eventual + * retryability, not a license to retry now — see {@link hotRetryUnsafe}. + */ + public readonly retryable = true + + /** + * An immediate, identical retry re-pays the FULL cost of the work that + * just timed out — it does not resume partway. Looping on this error + * (hot-retrying) repeats that full cost every attempt and can cascade + * into a CPU/resource storm on the caller's side. Callers MUST latch: on + * this error, record `{ at: Date.now(), error }`, surface one loud + * failure to their own caller, and hold a cooldown window before any + * re-attempt (clearing the latch only on success). Never retry this error + * in a tight loop. + */ + public readonly hotRetryUnsafe = true + constructor( timeoutMs: number, operationIndex: number, telemetry?: { + /** Milliseconds elapsed in the transaction when the budget tripped. */ elapsedMs?: number + /** Total number of operations in the batch that timed out. */ totalOperations?: number + /** Name of the operation the batch was about to start when it tripped, if named. */ operationName?: string } ) { @@ -93,8 +126,16 @@ export class TransactionTimeoutError extends TransactionError { telemetry?.elapsedMs !== undefined ? `${telemetry.elapsedMs}ms elapsed, ` : '' super( `Transaction timed out at operation ${progress}${name} — ${elapsed}budget ${timeoutMs}ms. ` + - `The batch rolled back atomically; retry with a higher timeoutMs or a smaller batch.`, - { timeoutMs, operationIndex, ...telemetry } + `The batch rolled back atomically; retryable after the underlying slowness resolves or ` + + `the budget is raised, but hot-retry-unsafe — latch and back off, never loop.`, + { + // Caller backoff inputs — all present on every instance: + /** Configured budget (ms) that was exceeded. */ + timeoutMs, + /** Index of the operation the batch was about to start when it tripped. */ + operationIndex, + ...telemetry + } ) this.name = 'TransactionTimeoutError' } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 8bede8d3..b5f286ed 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1658,8 +1658,10 @@ export interface BrainyConfig { * **start** — never whether already-completed work gets rolled back after * the fact (a single-op write can never time out post-hoc: it either runs * or it commits). A trip mid-batch still rolls back every applied operation - * atomically and throws a retryable `TransactionTimeoutError`; only the - * floor of the formula is configurable here. + * atomically and throws a `TransactionTimeoutError` that is + * retryable-with-latch, never hot-retry (see its `retryable` and + * `hotRetryUnsafe` fields); only the floor of the formula is configurable + * here. * * Raise this when a cold store's first writes after a restart legitimately * take longer than 30s per operation (e.g. page-cache-cold canonical writes diff --git a/tests/transaction/TransactionManager.unit.test.ts b/tests/transaction/TransactionManager.unit.test.ts index 86b1692c..29e7f7ae 100644 --- a/tests/transaction/TransactionManager.unit.test.ts +++ b/tests/transaction/TransactionManager.unit.test.ts @@ -5,7 +5,6 @@ * - High-level transaction API * - Statistics tracking * - Error handling - * - Result wrapping */ import { describe, it, expect, beforeEach } from 'vitest' @@ -84,42 +83,6 @@ describe('TransactionManager', () => { }) }) - describe('executeTransactionWithResult', () => { - it('should return detailed result', async () => { - const result = await manager.executeTransactionWithResult(async (tx) => { - tx.addOperation({ - execute: async () => { - await new Promise(resolve => setTimeout(resolve, 1)) - return async () => {} - } - }) - tx.addOperation({ execute: async () => undefined }) - return 'success' - }) - - expect(result.value).toBe('success') - expect(result.operationCount).toBe(2) - expect(result.executionTimeMs).toBeGreaterThanOrEqual(0) - }) - - it('should measure execution time', async () => { - const result = await manager.executeTransactionWithResult(async (tx) => { - tx.addOperation({ - execute: async () => { - await new Promise(resolve => setTimeout(resolve, 25)) - return async () => {} - } - }) - return 'done' - }) - - // Timer coalescing can fire a setTimeout up to a few ms EARLY under - // load, so assert well below the sleep — this tests that time is - // MEASURED, not the OS timer's precision. - expect(result.executionTimeMs).toBeGreaterThanOrEqual(20) - }) - }) - describe('Statistics Tracking', () => { it('should track total transactions', async () => { await manager.executeTransaction(async (tx) => { diff --git a/tests/unit/transaction/timeout-never-internally-retried.test.ts b/tests/unit/transaction/timeout-never-internally-retried.test.ts new file mode 100644 index 00000000..a43a81a8 --- /dev/null +++ b/tests/unit/transaction/timeout-never-internally-retried.test.ts @@ -0,0 +1,141 @@ +/** + * @module tests/unit/transaction/timeout-never-internally-retried + * @description Regression pin for the no-hot-retry contract (8.10.1). + * + * A production incident: a native-provider op ground 38-40s inside a + * transaction, blew the ~32s budget, was rolled back, and a CONSUMER pipeline + * hot-retried the identical operation into a 6-minute 100%-CPU storm. + * Investigation established brainy itself never auto-retries a + * `TransactionTimeoutError` — the storm was entirely the consumer's hot-retry + * loop, driven by a "retryable" doc-prose claim with no machine-readable + * contract. This file pins the brainy-side half of that story so it can never + * regress silently: + * + * (i) the underlying engine (`TransactionManager.executeTransaction()` → + * `Transaction.execute()`) — the exact machinery every single-record + * write (`add`/`update`/`remove`/...) drives via + * `Brainy.persistSingleOp()` — never internally re-executes a timed-out + * operation, and the error it surfaces carries `retryable === true` and + * `hotRetryUnsafe === true` (see `src/transaction/errors.ts`). + * + * Constructed directly (mirrors the existing + * `tests/unit/transaction/timeout-rollback.test.ts` pattern) rather than + * through a real `brain.add()` call: `transactTimeoutBudget()` floors + * every single-op write's budget at `opCount * 2000`ms with NO override + * seam (`transactionBudgetFloorMs` only RAISES that floor — it cannot + * lower it below the per-op-count term), so getting a real `add()` to + * time out requires a multi-second sleep. The engine-level + * `options.timeout` override used here is the exact same + * `TransactionManager`/`Transaction` code `persistSingleOp` calls — + * pinning it here pins add()'s guarantee without paying that wall-clock + * cost. + * + * (ii) `Brainy.add()`'s upsert-race retry loop (src/brainy.ts, + * `MAX_UPSERT_ATTEMPTS = 10`) — proving the loop's `catch` treats a + * `TransactionTimeoutError` as terminal (immediate rethrow) rather than + * the `InsertPreconditionExistsSignal` it retries on, so a mid-flight + * timeout can never be silently swallowed and re-attempted up to 10 + * times. + */ +import { describe, it, expect } from 'vitest' +import { TransactionManager } from '../../../src/transaction/TransactionManager.js' +import type { Operation, RollbackAction } from '../../../src/transaction/types.js' +import { TransactionTimeoutError } from '../../../src/transaction/errors.js' +import { Brainy } from '../../../src/brainy.js' +import { NounType } from '../../../src/types/graphTypes.js' + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) + +// Brainy's ValidationConfig fixes vectors at exactly 384 dimensions +// (src/utils/paramValidation.ts) — match it so add() doesn't reject test data. +const DIM = 384 +const V = (): number[] => Array(DIM).fill(0.1) + +/** An operation that counts every `execute()` invocation — the re-drive detector. */ +function countingOp(opts: { delayMs?: number; name: string }): Operation & { calls: number } { + const op = { + name: opts.name, + calls: 0, + async execute(): Promise { + op.calls++ + if (opts.delayMs) await sleep(opts.delayMs) + return async () => {} + } + } + return op +} + +describe('transaction timeouts are never internally re-driven (8.10.1 no-hot-retry contract)', () => { + it('(i) the single-op engine (TransactionManager.executeTransaction / Transaction.execute — what add() drives via persistSingleOp) runs the overrun operation EXACTLY once and surfaces ONE retryable+hotRetryUnsafe error', async () => { + const manager = new TransactionManager() + const op0 = countingOp({ name: 'op0-overruns-budget', delayMs: 30 }) + const op1 = countingOp({ name: 'op1-must-never-start' }) + + let caught: unknown + try { + await manager.executeTransaction( + async (tx) => { + tx.addOperation(op0) + tx.addOperation(op1) + }, + // Tiny explicit override — the same override seam `transact()` + // exposes as `options.timeoutMs`; wins outright over the + // opCount*2000 floor that gates every real single-op write + // (transactTimeoutBudget()'s override semantics). + { timeout: 5 } + ) + } catch (err) { + caught = err + } + + expect(caught).toBeInstanceOf(TransactionTimeoutError) + const err = caught as TransactionTimeoutError + // The machine-readable contract callers branch on instead of parsing + // message text (src/transaction/errors.ts). + expect(err.retryable).toBe(true) + expect(err.hotRetryUnsafe).toBe(true) + + // The re-drive assertion: op0 (the one that overran) executed EXACTLY + // once — nothing inside TransactionManager/Transaction looped back and + // re-ran it — and op1 never started at all (the budget gate stopped it + // before it began, per Transaction.execute()'s per-operation loop). + expect(op0.calls).toBe(1) + expect(op1.calls).toBe(0) + }) + + it('(ii) add()\'s upsert-race retry loop (MAX_UPSERT_ATTEMPTS=10) exits on the FIRST TransactionTimeoutError — attempt counter stays at 1, never mistaken for the lost-insert-race signal it retries on', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + await brain.init() + + let persistSingleOpCalls = 0 + const timeoutError = new TransactionTimeoutError(5, 1, { + elapsedMs: 6, + totalOperations: 2, + operationName: 'SaveNounMetadata' + }) + // Stub the private commit seam add() drives (persistSingleOp) to throw + // the exact error type the real engine surfaces on a mid-flight timeout. + // This test pins the upsert loop's EXCEPTION-HANDLING contract (does it + // retry a TransactionTimeoutError like it retries + // InsertPreconditionExistsSignal?), not the timing mechanics of a real + // timeout — those are pinned by test (i) and by + // tests/unit/transaction/timeout-rollback.test.ts. + ;(brain as any).persistSingleOp = async (): Promise => { + persistSingleOpCalls++ + throw timeoutError + } + + await expect( + brain.add({ data: 'a', type: NounType.Thing, vector: V() }) + ).rejects.toBe(timeoutError) + + // The loop's attempt counter: exactly one call, never retried up to + // MAX_UPSERT_ATTEMPTS. + expect(persistSingleOpCalls).toBe(1) + await brain.close() + }) +}) From 5b2cbf74e568d4bb1f89cd7019d8d70c05999188 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 24 Jul 2026 16:02:01 -0700 Subject: [PATCH 08/10] fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface A production deployment's warm report showed metadata: 'unavailable' under a native metadata provider. brain.warm()'s metadata leg only duck-typed the built-in JS manager's hydrateAll() method, which a native provider has no reason to implement. - MetadataIndexProvider (src/plugin.ts) gains an optional warm?(): Promise hook, mirroring the existing vector and graph provider hooks. brain.warm() now checks the active provider's own warm() FIRST, falls back to the JS manager's hydrateAll() when absent, and reports 'unavailable' only when neither exists -- never init() as a stand-in, since a native provider's init() may be a cheap verify rather than a real warm. - Tests (tests/unit/brainy/warm.test.ts): a live provider instance shaped to have warm() reports 'warmed' and the hook called with no hydrateAll fallback; shaped to have neither hook reports 'unavailable' (pins the honest branch); the unmodified built-in JS manager still reports 'warmed' via hydrateAll(), unchanged. Additive scope agreed mid-flight with the native-provider team: a maintenance-debt observability seam so an operator sees a grind coming instead of discovering it as a CPU storm. - New optional maintenanceDebt?(): Promise hook on all three provider contracts (vector, metadata, graph -- the same three warm?() lives on). ProviderMaintenanceDebt is fields-all-optional: a provider reports only what it truly measures (pendingBytes, pendingItems, lastPassCompletedAt, lastPassOutcome, converging), never an estimate dressed as fact. - New public brain.maintenanceDebt(): a pure passthrough -- for each surface it calls only the active provider's own hook and reports the payload verbatim, or 'unavailable' when absent. No thresholds, no polling, no JS-side estimation; the provider owns the numbers, the operator owns the policy. - ProviderMaintenanceDebt, MaintenanceDebtReport, and MaintenanceDebtOutcome are exported from the package root. - Tests (tests/unit/brainy/maintenance-debt.test.ts): hook present reports 'reported' with the exact payload passed through; hook absent reports 'unavailable' on every surface; mixed surfaces resolve independently of each other. RELEASES.md gains the 8.10.1 entry covering both fixes above and this feature, including the no-hot-retry contract from the prior commit. --- RELEASES.md | 62 +++++++++++ src/brainy.ts | 112 ++++++++++++++++++- src/index.ts | 10 ++ src/plugin.ts | 83 ++++++++++++++ tests/unit/brainy/maintenance-debt.test.ts | 122 +++++++++++++++++++++ tests/unit/brainy/warm.test.ts | 88 +++++++++++++++ 6 files changed, 471 insertions(+), 6 deletions(-) create mode 100644 tests/unit/brainy/maintenance-debt.test.ts diff --git a/RELEASES.md b/RELEASES.md index 89bf38e7..03d7283a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,68 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v8.10.1 — 2026-07-24 (the no-hot-retry contract + warm()'s metadata surface under native providers) + +From a production incident: a native-provider op ground 38-40s inside a transaction, +blew the ~32s apply-phase budget, was rolled back (zero loss, by design), and a +downstream pipeline hot-retried the identical operation into a 6-minute, 100%-CPU +storm. Investigation confirmed Brainy itself never auto-retries a timed-out +transaction — the storm was entirely the consumer's own retry loop, driven by a +"retryable" doc-prose claim with no machine-readable contract to branch on. This +release closes that contract gap and, separately, fixes a real `warm()` reporting gap +surfaced by the same investigation. + +- **`TransactionTimeoutError` is now a machine-readable no-hot-retry contract.** Two + new typed, always-`true` fields replace prose-only guidance: + - `retryable: true` — the operation MAY succeed on a later attempt, once the + underlying slowness resolves or the budget is deliberately raised + (`transactionBudgetFloorMs`, or a batch's own `timeoutMs` override). + - `hotRetryUnsafe: true` — an immediate, identical retry re-pays the FULL cost of + the work that just timed out (it does not resume partway) and can cascade into + exactly the CPU storm above. **Never loop on this error.** The documented pattern + is a latch, not a retry loop: + ``` + on TransactionTimeoutError: + record { at: Date.now(), error } + rethrow loudly to your own caller + hold a cooldown window before any re-attempt + clear the latch only on a subsequent success + ``` + - `context` (unchanged, now fully documented) carries the backoff inputs: + `timeoutMs`, `operationIndex`, `elapsedMs`, `totalOperations`, `operationName`. + - Every "retryable" doc-prose site referencing this error (`transact()`'s + `timeoutMs` option, `transactionBudgetFloorMs`, `Transaction.execute()`) now + points at these fields instead of bare prose. + - Regression-pinned: the engine never internally re-drives a timed-out operation + (verified via an execution counter through both the single-op write path and + `add()`'s upsert-race retry loop), so this has always been true — it is now + provable and typed. +- **Dead code removed**: `TransactionManager.executeTransactionWithResult()` had zero + callers in this codebase and is deleted. +- **`brain.warm()`'s metadata surface now routes through the ACTIVE provider.** A + production deployment's warm report showed `metadata: 'unavailable'` under a native + metadata provider — the previous logic only duck-typed the built-in JS manager's + `hydrateAll()` method, which a native provider has no reason to implement. The + metadata provider contract (`MetadataIndexProvider`, `src/plugin.ts`) gains an + optional `warm?(): Promise` hook, mirroring the existing vector and graph + provider hooks. `brain.warm()` now checks the active provider's own `warm()` FIRST, + falls back to the JS manager's `hydrateAll()` when absent, and only reports + `'unavailable'` when neither exists — never `init()` as a stand-in, since a native + provider's `init()` may be a cheap verify rather than a real warm. A native + provider lights this surface up the same way `@soulcraft/cor` already lights the + vector and graph surfaces: implement `warm()` on its metadata provider. +- **New: `brain.maintenanceDebt()`** — the observability seam so an operator sees a + provider's outstanding background maintenance work (pending bytes/items, last pass + outcome, whether it's converging) BEFORE it grinds into the kind of budget-busting + op this release's timeout contract exists for, instead of discovering it as a CPU + storm. It is a pure passthrough: brainy applies no thresholds, no polling, and no + estimation — it calls each active provider's own optional `maintenanceDebt?()` hook + (vector, metadata, graph — the same three contracts `warm?()` lives on) and reports + the payload verbatim, or `'unavailable'` when a surface's provider doesn't track + debt. Useful as a pre-warm/post-warm check or a boot gate. `@soulcraft/cor` does not + yet implement the hook as of this release — expect it on cor's next release; until + then all three surfaces honestly report `'unavailable'`. + ## Unreleased (the warm contract: cold-restart writes stop paying demand-load latency) From a production deployment's cold-restart incident: the FIRST writes after every diff --git a/src/brainy.ts b/src/brainy.ts index 37b6e491..007cdb32 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -65,7 +65,8 @@ import type { OpaqueIdSet, AtGenerationVectors, VectorIndexProvider, - GraphIndexProvider + GraphIndexProvider, + ProviderMaintenanceDebt } from './plugin.js' import type { BrainyPlugin, @@ -424,6 +425,30 @@ export interface WarmReport { totalDurationMs: number } +/** + * @description Result of {@link Brainy.maintenanceDebt}: one outcome per + * index surface, mirroring {@link WarmReport}'s shape. + * - `'reported'` — the active provider for this surface implements + * `maintenanceDebt?()` and its {@link ProviderMaintenanceDebt} payload is + * attached verbatim under `debt`. + * - `'unavailable'` — the active provider does not implement the hook, so + * nothing is known; brainy never estimates or infers a payload on its + * behalf. + */ +export type MaintenanceDebtOutcome = 'reported' | 'unavailable' + +/** + * @description Per-surface result of {@link Brainy.maintenanceDebt}. Brainy + * performs no thresholding, polling, or estimation over this data — it is a + * pure passthrough of each active provider's own self-report (the provider + * owns the numbers; the operator owns the policy). + */ +export interface MaintenanceDebtReport { + vector: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt } + metadata: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt } + graph: { outcome: MaintenanceDebtOutcome; debt?: ProviderMaintenanceDebt } +} + /** * How long a failed aggregation-backfill walk suppresses fresh walk attempts. * Within the window, queries rethrow the recorded failure instantly (loud, @@ -14313,9 +14338,16 @@ export class Brainy implements BrainyInterface { * *some* backing storage as a side effect but is reported honestly as * `'probed'`, never `'warmed'`. An empty index or unknown dimension has * nothing to probe (`'unavailable'`). - * - **Metadata**: full hydration — every persisted field's sparse index is - * loaded from storage (`MetadataIndexManager.hydrateAll()`), not just the - * heuristic common-fields subset `init()` warms. + * - **Metadata**: calls the provider's own `warm?()` when the active + * `'metadataIndex'` provider implements it (`'warmed'`) — the seam a + * native metadata provider lights up so it is not duck-typed against the + * JS manager's method. Otherwise falls back to full hydration on the + * built-in JS manager — every persisted field's sparse index is loaded + * from storage (`MetadataIndexManager.hydrateAll()`), not just the + * heuristic common-fields subset `init()` warms — and reports `'warmed'`. + * Neither seam present → `'unavailable'` (honest: `init()` is never used + * as a substitute here, since a native provider's `init()` may be a + * cheap verify rather than a real warm). * - **Graph**: calls the provider's own `warm?()` when the active graph * provider implements it; otherwise re-runs its existing eager cold-load * `init()` seam (idempotent — the JS adjacency index's `init()` already @@ -14371,12 +14403,23 @@ export class Brainy implements BrainyInterface { // --- Metadata -------------------------------------------------------- const metadataStart = Date.now() let metadataOutcome: WarmOutcome + const metadataProvider = this.metadataIndex as unknown as MetadataIndexProvider const metadataWithHydrate = this.metadataIndex as unknown as { hydrateAll?: () => Promise } - if (typeof metadataWithHydrate.hydrateAll === 'function') { + if (typeof metadataProvider.warm === 'function') { + // Active provider (e.g. a native metadata index) declares its own warm + // seam — route through it FIRST so a native provider's warmth is + // reported honestly instead of being duck-typed against the JS + // manager's hydrateAll(), which a native provider does not implement. + await metadataProvider.warm() + metadataOutcome = 'warmed' + } else if (typeof metadataWithHydrate.hydrateAll === 'function') { + // Built-in JS manager path — full sparse-index hydration. await metadataWithHydrate.hydrateAll() metadataOutcome = 'warmed' } else { - // No hydration seam on this metadata provider — nothing to run. + // No hydration seam on this metadata provider — nothing to run. (No + // init() fallback here: init() on a native provider may be a cheap + // verify, and reporting that as warmth would lie.) metadataOutcome = 'unavailable' } const metadataDurationMs = Date.now() - metadataStart @@ -14410,6 +14453,63 @@ export class Brainy implements BrainyInterface { } } + /** + * Read each index surface's self-reported outstanding maintenance work — + * the observability seam so an operator sees a grind coming (rising + * pending bytes/items, a stalled background pass) instead of discovering + * it as a CPU storm or a transaction blowing its budget mid-flight (see + * {@link TransactionTimeoutError}). + * + * PURE PASSTHROUGH: for each of vector/metadata/graph, this calls ONLY the + * ACTIVE provider's own `maintenanceDebt?()` hook (the same per-surface + * provider resolution {@link Brainy.warm} uses) and reports its + * {@link ProviderMaintenanceDebt} payload verbatim. There is no JS-side + * fallback computation, no threshold evaluation, and no polling — brainy + * surfaces the truth the provider measured; the provider owns the numbers + * and the operator owns the policy (what threshold matters, what action to + * take). A surface whose active provider does not implement the hook + * reports `'unavailable'` — never a guessed or zeroed payload. + * + * @returns A {@link MaintenanceDebtReport}: per-surface outcome + payload. + * @example + * ```typescript + * const debt = await brain.maintenanceDebt() + * if (debt.metadata.outcome === 'reported' && debt.metadata.debt?.pendingBytes) { + * console.log('metadata pending bytes:', debt.metadata.debt.pendingBytes) + * } + * ``` + */ + async maintenanceDebt(): Promise { + await this.ensureInitialized({ needs: ['vector', 'metadata', 'graph'] }) + + // --- Vector --------------------------------------------------------- + const vectorProvider = this.index as VectorIndexProvider & { + maintenanceDebt?: () => Promise + } + const vector = + typeof vectorProvider.maintenanceDebt === 'function' + ? { outcome: 'reported' as const, debt: await vectorProvider.maintenanceDebt() } + : { outcome: 'unavailable' as const } + + // --- Metadata -------------------------------------------------------- + const metadataProvider = this.metadataIndex as unknown as MetadataIndexProvider + const metadata = + typeof metadataProvider.maintenanceDebt === 'function' + ? { outcome: 'reported' as const, debt: await metadataProvider.maintenanceDebt() } + : { outcome: 'unavailable' as const } + + // --- Graph ------------------------------------------------------------- + const graphProvider = this.graphIndex as GraphIndexProvider & { + maintenanceDebt?: () => Promise + } + const graph = + typeof graphProvider.maintenanceDebt === 'function' + ? { outcome: 'reported' as const, debt: await graphProvider.maintenanceDebt() } + : { outcome: 'unavailable' as const } + + return { vector, metadata, graph } + } + /** * Explicitly warm up the embedding engine * diff --git a/src/index.ts b/src/index.ts index 00adc191..e60739f7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,6 +31,11 @@ export type { DiagnosticsResult } from './brainy.js' // brain.warm() — eager index/storage readiness report (per-surface honest // outcome + timing). See the WarmReport JSDoc in brainy.ts. export type { WarmReport, WarmOutcome } from './brainy.js' +// brain.maintenanceDebt() — per-surface passthrough of each active +// provider's self-reported background maintenance debt. See the +// MaintenanceDebtReport JSDoc in brainy.ts and ProviderMaintenanceDebt in +// plugin.ts for the measure-only-what-you-track contract. +export type { MaintenanceDebtReport, MaintenanceDebtOutcome } from './brainy.js' export type { GraphAuditReport, GraphAuditDiscrepancy @@ -227,6 +232,11 @@ export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.j export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js' +// Optional provider self-report of outstanding background maintenance work +// (compaction, deferred writes, etc.) — the payload type for +// brain.maintenanceDebt(). See the measure-only-what-you-track contract on +// ProviderMaintenanceDebt in plugin.ts. +export type { ProviderMaintenanceDebt } from './plugin.js' // Optional native graph-acceleration engine (cor 3.0) — the published provider // contract + its columnar wire types. Brainy feature-detects an implementation // and falls back to its pure-TS adjacency when absent. diff --git a/src/plugin.ts b/src/plugin.ts index 02639955..ce973386 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -171,6 +171,37 @@ export interface ProviderInvariantReport { durationMs: number } +/** + * @description A provider's self-report of its own outstanding background + * maintenance work (compaction, deferred writes, a build-new→verify→swap in + * flight, etc.) — the observability seam so an operator sees a grind coming + * (rising pending bytes/items, a stalled pass) instead of discovering it as a + * CPU storm or a timeout under transaction budget pressure. Every field is + * OPTIONAL and every field is a MEASUREMENT: a provider reports ONLY what it + * actually tracks, never an estimate dressed up as a fact. Absence of the + * {@link VectorIndexProvider.maintenanceDebt} / + * {@link GraphIndexProvider.maintenanceDebt} / + * {@link MetadataIndexProvider.maintenanceDebt} hook itself means the + * provider does not track debt at all — brainy reports that surface + * `'unavailable'` rather than inventing zeros. Brainy performs NO threshold + * checks, NO polling, and NO JS-side estimation over this payload — it is a + * pure passthrough via {@link Brainy.maintenanceDebt}; the provider owns the + * numbers and the operator owns the policy (what threshold matters, what to + * do about it). + */ +export interface ProviderMaintenanceDebt { + /** Bytes of outstanding/unmerged work, if the provider measures it (e.g. unflushed writes, unmerged segments). */ + pendingBytes?: number + /** Count of outstanding items (records, segments, nodes) awaiting the provider's background pass. */ + pendingItems?: number + /** Epoch millis when the provider's last maintenance pass finished, if it tracks one. */ + lastPassCompletedAt?: number + /** How the last pass ended, if the provider tracks pass outcomes. */ + lastPassOutcome?: 'completed' | 'partial' | 'failed' + /** `true` if the provider's own measurements show debt trending down (making progress); `false` if flat or growing; omitted if the provider can't tell. */ + converging?: boolean +} + /** * The `'metadataIndex'` provider — a drop-in for `MetadataIndexManager`. * Brainy calls this surface via `this.metadataIndex.*` (see `brainy.ts`) and @@ -181,6 +212,34 @@ export interface MetadataIndexProvider { flush(): Promise rebuild(): Promise + /** + * @description OPTIONAL. Eagerly load/fault-in backing storage (e.g. mmap + * pretouch, full sparse-index hydration) so first queries run at + * steady-state cost. Optional; absence means the provider demand-loads. + * Mirrors {@link GraphIndexProvider.warm} / the vector provider's `warm?()` + * (`src/plugin.ts` VectorIndexProvider). Distinct from `init()`: `init` is + * required and runs once automatically during brain startup; `warm` is a + * separate, explicit step a caller opts into via `brain.warm()` (or + * `warmOnOpen`) to pre-pay demand-load cost `init` left lazy. Idempotent — + * calling it more than once must be safe and cheap on a brain that is + * already warm. A provider that already loads everything eagerly in + * `init()` may implement `warm` as a no-op or omit it — `brain.warm()` + * falls back to the built-in JS manager's `hydrateAll()` duck-type when + * absent, and to an honest `'unavailable'` when neither exists. + */ + warm?(): Promise + + /** + * @description OPTIONAL self-reported {@link ProviderMaintenanceDebt} — + * the observability seam so an operator sees outstanding background + * maintenance work (e.g. unmerged postings) BEFORE it grinds a transaction + * into a budget-busting op. Absence means this provider does not track + * debt; `brain.maintenanceDebt()` reports this surface `'unavailable'` + * rather than guessing. See {@link ProviderMaintenanceDebt} for the + * measure-only-what-you-track contract. + */ + maintenanceDebt?(): Promise + /** * @description OPTIONAL honest durability signal (readiness contract, * mirrors `isReady?()` on the graph and vector providers). `true` ⇔ the @@ -395,6 +454,18 @@ export interface GraphIndexProvider { */ warm?(): Promise + /** + * @description OPTIONAL self-reported {@link ProviderMaintenanceDebt} — + * the observability seam so an operator sees outstanding background + * maintenance work (e.g. a build-new→verify→swap in flight, unmerged + * adjacency segments) BEFORE it grinds a transaction into a + * budget-busting op. Absence means this provider does not track debt; + * `brain.maintenanceDebt()` reports this surface `'unavailable'` rather + * than guessing. See {@link ProviderMaintenanceDebt} for the + * measure-only-what-you-track contract. + */ + maintenanceDebt?(): Promise + /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background @@ -1057,6 +1128,18 @@ export interface VectorIndexProvider { */ warm?(): Promise + /** + * @description OPTIONAL self-reported {@link ProviderMaintenanceDebt} — + * the observability seam so an operator sees outstanding background + * maintenance work (e.g. unflushed writes, a pending rebuild) BEFORE it + * grinds a transaction into a budget-busting op. Absence means this + * provider does not track debt; `brain.maintenanceDebt()` reports this + * surface `'unavailable'` rather than guessing. See + * {@link ProviderMaintenanceDebt} for the measure-only-what-you-track + * contract. + */ + maintenanceDebt?(): Promise + /** * @description OPTIONAL honest durability signal (readiness contract, * mirrors {@link GraphIndexProvider.isReady}). `true` ⇔ the persisted diff --git a/tests/unit/brainy/maintenance-debt.test.ts b/tests/unit/brainy/maintenance-debt.test.ts new file mode 100644 index 00000000..4026d655 --- /dev/null +++ b/tests/unit/brainy/maintenance-debt.test.ts @@ -0,0 +1,122 @@ +/** + * @module tests/unit/brainy/maintenance-debt + * @description Coverage for `brain.maintenanceDebt()` (8.10.1) — the + * observability seam so an operator sees a provider's outstanding background + * maintenance work (compaction, deferred writes, a build-new→verify→swap in + * flight, ...) BEFORE it grinds a transaction into a budget-busting op, the + * same failure class documented on `TransactionTimeoutError` + * (src/transaction/errors.ts). Sibling to tests/unit/brainy/warm.test.ts, + * which establishes this file's technique: shape the probe points brain.ts + * reads (`typeof provider.maintenanceDebt === 'function'`) directly on the + * REAL, live provider instances rather than hand-rolling full fakes for the + * larger `MetadataIndexProvider` / `GraphIndexProvider` interfaces. + * + * `brain.maintenanceDebt()` is a PURE PASSTHROUGH: no thresholds, no + * polling, no JS-side estimation — these tests pin exactly that by asserting + * the returned payload is the provider's object, verbatim. + */ +import { describe, it, expect } from 'vitest' +import { Brainy } from '../../../src/brainy.js' +import { NounType } from '../../../src/types/graphTypes.js' +import type { ProviderMaintenanceDebt } from '../../../src/plugin.js' + +// Brainy's ValidationConfig fixes vectors at exactly 384 dimensions +// (src/utils/paramValidation.ts) — match it so add() doesn't reject test data. +const DIM = 384 +const V = (seed = 1): number[] => Array.from({ length: DIM }, (_, i) => Math.sin(seed + i)) + +async function freshBrain(): Promise> { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) }) + return brain +} + +describe('brain.maintenanceDebt()', () => { + it('reports "unavailable" for every surface when no active provider implements maintenanceDebt() (the built-in JS stack today)', async () => { + const brain = await freshBrain() + + const report = await brain.maintenanceDebt() + + expect(report.vector.outcome).toBe('unavailable') + expect(report.vector.debt).toBeUndefined() + expect(report.metadata.outcome).toBe('unavailable') + expect(report.metadata.debt).toBeUndefined() + expect(report.graph.outcome).toBe('unavailable') + expect(report.graph.debt).toBeUndefined() + + await brain.close() + }) + + it('reports "reported" + the exact payload when the active provider implements maintenanceDebt() (verbatim passthrough, no thresholding)', async () => { + const brain = await freshBrain() + + const vectorDebt: ProviderMaintenanceDebt = { + pendingBytes: 4_096, + pendingItems: 12, + lastPassCompletedAt: 1_700_000_000_000, + lastPassOutcome: 'completed', + converging: true + } + ;(brain as any).index.maintenanceDebt = async () => vectorDebt + + const report = await brain.maintenanceDebt() + + expect(report.vector.outcome).toBe('reported') + // Verbatim passthrough — the exact object, not a re-derived copy. + expect(report.vector.debt).toBe(vectorDebt) + // Untouched surfaces stay honestly 'unavailable'. + expect(report.metadata.outcome).toBe('unavailable') + expect(report.graph.outcome).toBe('unavailable') + + await brain.close() + }) + + it('mixed surfaces: each surface\'s outcome depends ONLY on its OWN active provider — one surface reporting never leaks into another', async () => { + const brain = await freshBrain() + + const metadataDebt: ProviderMaintenanceDebt = { + pendingItems: 3, + lastPassOutcome: 'partial', + converging: false + } + const graphDebt: ProviderMaintenanceDebt = { + pendingBytes: 0, + converging: true + } + ;(brain as any).metadataIndex.maintenanceDebt = async () => metadataDebt + ;(brain as any).graphIndex.maintenanceDebt = async () => graphDebt + // Vector is deliberately left unpatched. + + const report = await brain.maintenanceDebt() + + expect(report.vector.outcome).toBe('unavailable') + expect(report.vector.debt).toBeUndefined() + + expect(report.metadata.outcome).toBe('reported') + expect(report.metadata.debt).toBe(metadataDebt) + + expect(report.graph.outcome).toBe('reported') + expect(report.graph.debt).toBe(graphDebt) + + await brain.close() + }) + + it('an empty ProviderMaintenanceDebt object (every field omitted) is still honestly "reported" — presence of the hook, not the payload\'s richness, drives the outcome', async () => { + const brain = await freshBrain() + + const emptyDebt: ProviderMaintenanceDebt = {} + ;(brain as any).graphIndex.maintenanceDebt = async () => emptyDebt + + const report = await brain.maintenanceDebt() + + expect(report.graph.outcome).toBe('reported') + expect(report.graph.debt).toEqual({}) + + await brain.close() + }) +}) diff --git a/tests/unit/brainy/warm.test.ts b/tests/unit/brainy/warm.test.ts index ce213696..8a0bb4da 100644 --- a/tests/unit/brainy/warm.test.ts +++ b/tests/unit/brainy/warm.test.ts @@ -307,4 +307,92 @@ describe('brain.warm()', () => { expect(report.totalDurationMs).toBeGreaterThanOrEqual(0) await brain.close() }) + + // --- Metadata leg routes through the ACTIVE provider (8.10.1) ----------- + // + // `MetadataIndexProvider` is a ~50-method interface (src/plugin.ts) — far + // too large to hand-write a compliant fake class the way `FakeVectorProvider` + // fakes the ~8-method `VectorIndexProvider` above. Test (c) already + // establishes this file's pattern for the metadata leg: exercise the REAL + // `MetadataIndexManager` instance and shape just the probe points brain.ts + // reads (`typeof provider.warm === 'function'` / + // `typeof provider.hydrateAll === 'function'`) directly on that instance. + // Shadowing an own property on the live object stands in for "a different + // provider implementation" without needing a hand-rolled full fake — the + // rest of the real manager (used by add()/init() above) is untouched. + describe('metadata leg — warm() routes through the active provider', () => { + it('(f) calls the ACTIVE metadata provider\'s warm() when present and reports "warmed", never falling back to hydrateAll', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) }) + + const metadataIndex = (brain as any).metadataIndex + let warmCalls = 0 + let hydrateAllCalls = 0 + const origHydrateAll = metadataIndex.hydrateAll.bind(metadataIndex) + metadataIndex.hydrateAll = async (...args: unknown[]) => { + hydrateAllCalls++ + return origHydrateAll(...args) + } + // Simulates a native metadata provider declaring the optional `warm()` + // hook added to `MetadataIndexProvider` (src/plugin.ts) in 8.10.1. + metadataIndex.warm = async () => { + warmCalls++ + } + + const report = await brain.warm() + + expect(warmCalls).toBe(1) + expect(hydrateAllCalls).toBe(0) // warm() ran — no hydrateAll fallback + expect(report.metadata.outcome).toBe('warmed') + await brain.close() + }) + + it('reports "unavailable" when the active metadata provider implements neither warm() nor hydrateAll() (the honest branch a native provider without either hook must hit)', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) }) + + const metadataIndex = (brain as any).metadataIndex + // Shadow away BOTH optional hooks — models a genuinely native provider + // that (unlike the built-in JS manager) offers neither seam. This must + // never fall back to calling init() as a stand-in for warmth. + metadataIndex.warm = undefined + metadataIndex.hydrateAll = undefined + + const report = await brain.warm() + + expect(report.metadata.outcome).toBe('unavailable') + await brain.close() + }) + + it('the built-in JS manager (no warm()) still reports "warmed" via its existing hydrateAll() duck-type — unchanged by the new provider hook', async () => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, vector: V(1) }) + + // No patching at all — the default built-in MetadataIndexManager has + // hydrateAll() but no warm(), exactly as it did before this change. + const metadataIndex = (brain as any).metadataIndex + expect(typeof metadataIndex.warm).not.toBe('function') + expect(typeof metadataIndex.hydrateAll).toBe('function') + + const report = await brain.warm() + + expect(report.metadata.outcome).toBe('warmed') + await brain.close() + }) + }) }) From edf123a5e232919881ae9d5bfaa4877c7ee457ee Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 24 Jul 2026 16:04:41 -0700 Subject: [PATCH 09/10] refactor: remove the orphaned transaction-result type left behind by the dead-path removal --- src/transaction/types.ts | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/transaction/types.ts b/src/transaction/types.ts index 9a3a2eaa..6cbc56ca 100644 --- a/src/transaction/types.ts +++ b/src/transaction/types.ts @@ -66,26 +66,6 @@ export interface TransactionContext { */ export type TransactionFunction = (ctx: TransactionContext) => Promise -/** - * Transaction execution result - */ -export interface TransactionResult { - /** - * Result value from user function - */ - value: T - - /** - * Number of operations executed - */ - operationCount: number - - /** - * Execution time in milliseconds - */ - executionTimeMs: number -} - /** * Transaction execution options */ From d9cc7b9024aff3fbffeb2fee658543d81fb4c0c9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 24 Jul 2026 16:09:47 -0700 Subject: [PATCH 10/10] chore(release): 8.10.1 --- CHANGELOG.md | 8 ++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6dd91fd..a5f344cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) + +- refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) +- fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) +- fix: transaction timeouts are a typed no-hot-retry contract; engine-side non-retry pinned; dead transaction path removed (003e2a74) +- chore: the forge is the address — retire the archived mirror from every live surface (22702b81) + + ### [8.10.0](https://github.com/soulcraftlabs/brainy/compare/v8.9.0...v8.10.0) (2026-07-23) - docs: adoption storefront — contributing guide, security policy, README support + cor section (9a99a7b) diff --git a/package-lock.json b/package-lock.json index 37aeb81d..d0c7b9d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "8.10.0", + "version": "8.10.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "8.10.0", + "version": "8.10.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index a3ece83c..ce670369 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "8.10.0", + "version": "8.10.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js",