feat(reprojection): the one doors-open machinery — budget-capped, yielding, foreground-preempted, atomic-swap; poison records quarantine typed

The generic reprojection engine (pure TS; the twin of the native
implementation — same frozen contract, one shared conformance intent):
register any ProjectionAdapter; advance(family, {budgetMs}) folds facts
from the adapter's own watermark to the head in installments ≤50ms with
real macrotask yields; foreground door traffic bumps the DoorSignal and
an in-flight advance yields within one installment ('preempted');
advanceAll round-robins families fairly. swap(family, buildAdapter) is
the doors-open migration primitive: the OLD projection keeps serving
while the new one builds beside it, the flip is atomic at parity, and a
concurrent second swap refuses typed. A fact the fold cannot apply
(typed ProjectionApplyError) is QUARANTINED — skipped, ledgered,
narrated per-doubling, exposed for refuse-affected-reads — the service
class law's fourth answer: never a wedged rebuild, never a silent skip.
The engine never writes stamps: each adapter owns its durability and its
stamp-after-data discipline. Upgrade, heal, and rebuild are now the same
machinery behind open doors.

FactLogSource wires any host's fact scan in one line
(factSourceFromHost(brain)); window-contract violations are loud.

Pins: 23 unit (budget resume without refold · preemption within one
installment · round-robin fairness under a skewed backlog · build-beside
visibility mid-swap · atomic flip · single-flight refusal · quarantine
skip/ledger/doubling · non-typed throw aborts · losing adapter
discarded) + 3 integration on a real brain (fold matches ground truth ·
doors answer mid-fold with the preemption path exercised · crash
mid-fold resumes from the stamp, never refolds).

Gates: unit 2054/2054 (157 files) · integration 820 (93 files) ·
conformance 31/31.

tests/integration/reprojection-doors-open.test.ts

@@ -0,0 +1,257 @@
+/**
+ * @module tests/integration/reprojection-doors-open
+ * @description The reprojection engine against a REAL brain on filesystem
+ * storage: a toy secondary projection (bucket counts with its own watermark
+ * artifact, stamp-after-data per src/utils/projectionWatermark.ts) folds the
+ * brain's committed facts through the engine, wired with the callback-form
+ * {@link FactLogSource} over `brain.scanFacts`.
+ *
+ * Proves the three doors-open rows:
+ * (i)   folding to caught-up matches ground-truth counts;
+ * (ii)  mid-fold, `find()` and `get()` still answer, and a door bump
+ *       preempts the advance at the next boundary (mechanism-pinned via
+ *       batch counts, not wall-clock);
+ * (iii) a crash mid-fold (abandon; reopen; re-advance) resumes from the
+ *       durable stamp — never refolds from zero.
+ */
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync, existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { Brainy } from '../../src/index.js'
+import {
+  ReprojectionEngine,
+  type ProjectionAdapter
+} from '../../src/reprojection/reprojectionEngine.js'
+import { FactLogSource } from '../../src/reprojection/factLogSource.js'
+import { makeProjectionStamp, readStampedWatermark } from '../../src/utils/projectionWatermark.js'
+import type { CommitFact } from '../../src/db/factLog.js'
+
+/** 50 rows, 5 buckets, 10 each. */
+const ROWS = 50
+const BUCKETS = 5
+const GROUND_TRUTH: Record<string, number> = { b0: 10, b1: 10, b2: 10, b3: 10, b4: 10 }
+
+/**
+ * The toy secondary projection: latest bucket per entity id, persisted as a
+ * data file plus a SEPARATE stamp artifact written stamp-after-data via the
+ * shared projectionWatermark helpers. Idempotent by construction (latest-
+ * state per id), so at-least-once redelivery on resume is harmless.
+ */
+class BucketCountProjection implements ProjectionAdapter {
+  readonly family = 'bucket-counts'
+  /** Every generation this INSTANCE applied — the refold detector for (iii). */
+  readonly appliedGenerations: number[] = []
+  private latest: Map<string, string | null>
+  private wm: number | null
+
+  private constructor(
+    private readonly dir: string,
+    wm: number | null,
+    latest: Map<string, string | null>
+  ) {
+    this.wm = wm
+    this.latest = latest
+  }
+
+  /** Load from the artifact dir — data is trusted only under a valid stamp. */
+  static async open(dir: string): Promise<BucketCountProjection> {
+    mkdirSync(dir, { recursive: true })
+    const stampPath = join(dir, 'stamp.json')
+    const dataPath = join(dir, 'data.json')
+    let wm: number | null = null
+    if (existsSync(stampPath)) {
+      wm = readStampedWatermark(JSON.parse(readFileSync(stampPath, 'utf8')))
+    }
+    const latest = new Map<string, string | null>(
+      wm !== null && existsSync(dataPath)
+        ? (JSON.parse(readFileSync(dataPath, 'utf8')) as Array<[string, string | null]>)
+        : []
+    )
+    return new BucketCountProjection(dir, wm, latest)
+  }
+
+  /** Non-null bucket tallies from the latest-state map. */
+  counts(): Record<string, number> {
+    const out: Record<string, number> = {}
+    for (const bucket of this.latest.values()) {
+      if (bucket !== null) out[bucket] = (out[bucket] ?? 0) + 1
+    }
+    return out
+  }
+
+  watermark(): number | null {
+    return this.wm
+  }
+
+  async applyBatch(facts: CommitFact[], upTo: number): Promise<void> {
+    for (const fact of facts) {
+      this.appliedGenerations.push(fact.generation)
+      for (const op of fact.ops) {
+        if (op.kind !== 'noun') continue
+        if (op.record === null) {
+          this.latest.set(op.id, null) // tombstone
+          continue
+        }
+        // The stored noun record nests user metadata under `.metadata`.
+        const stored = op.record.metadata as Record<string, unknown> | null
+        const user = (stored?.metadata ?? stored) as Record<string, unknown> | null
+        const bucket = typeof user?.bucket === 'string' ? user.bucket : null
+        this.latest.set(op.id, bucket)
+      }
+    }
+    // Durability THEN stamp — the projectionWatermark law.
+    writeFileSync(join(this.dir, 'data.json'), JSON.stringify([...this.latest]))
+    writeFileSync(join(this.dir, 'stamp.json'), JSON.stringify(makeProjectionStamp(upTo)))
+    this.wm = upTo
+  }
+
+  async discard(): Promise<void> {
+    rmSync(this.dir, { recursive: true, force: true })
+  }
+}
+
+describe('reprojection doors-open — a real brain, a toy secondary projection', () => {
+  let brainDir: string
+  let projRoot: string
+  let brain: Brainy
+  const ids: string[] = []
+
+  const openBrain = async (dir: string): Promise<Brainy> => {
+    const b = new Brainy({
+      storage: { type: 'filesystem', path: dir },
+      requireSubtype: false,
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  /**
+   * The production wiring, callback form: the engine's `from` is an EXCLUSIVE
+   * lower bound, `scanFacts` bounds are inclusive — hence `from + 1`; the
+   * first batch is returned and the handle closed (short batches at segment
+   * boundaries are legal — only EMPTY means caught up).
+   */
+  const sourceFor = (b: Brainy): FactLogSource =>
+    new FactLogSource(async (from, limit) => {
+      const scan = b.scanFacts({ fromGeneration: from + 1, batchSize: limit })
+      if (!scan) throw new Error('this brain hosts no fact log — cannot reproject')
+      const iterator = scan.batches()
+      try {
+        const first = await iterator.next()
+        return first.done ? [] : first.value.facts
+      } finally {
+        if (typeof iterator.return === 'function') await iterator.return(undefined)
+      }
+    })
+
+  beforeAll(async () => {
+    brainDir = mkdtempSync(join(tmpdir(), 'brainy-reproj-'))
+    projRoot = mkdtempSync(join(tmpdir(), 'brainy-reproj-artifacts-'))
+    brain = await openBrain(brainDir)
+    for (let i = 0; i < ROWS; i++) {
+      ids.push(
+        await brain.add({
+          data: `record ${i} filed in bucket ${i % BUCKETS}`,
+          type: 'document',
+          metadata: { bucket: `b${i % BUCKETS}` }
+        })
+      )
+    }
+  }, 240_000)
+
+  afterAll(async () => {
+    await brain?.close().catch(() => {})
+    rmSync(brainDir, { recursive: true, force: true })
+    rmSync(projRoot, { recursive: true, force: true })
+  })
+
+  it('(i) folds to caught-up through the engine and matches ground-truth counts', async () => {
+    const projection = await BucketCountProjection.open(join(projRoot, 'i'))
+    const engine = new ReprojectionEngine({ source: sourceFor(brain), batchSize: 8 })
+    engine.register(projection)
+
+    const result = await engine.advance(projection.family, { budgetMs: 60_000 })
+
+    expect(result.status).toBe('caught-up')
+    expect(result.watermark).toBeGreaterThanOrEqual(ROWS) // one generation per add, at least
+    expect(result.applied).toBeGreaterThanOrEqual(ROWS)
+    expect(engine.quarantined(projection.family)).toEqual([])
+    expect(projection.counts()).toEqual(GROUND_TRUTH)
+    // The stamp on disk is the adapter's own — stamped exactly at the fold head.
+    const reloaded = await BucketCountProjection.open(join(projRoot, 'i'))
+    expect(reloaded.watermark()).toBe(result.watermark)
+    expect(reloaded.counts()).toEqual(GROUND_TRUTH)
+  })
+
+  it('(ii) doors stay open mid-fold: find() and get() answer, and a bump preempts the advance', async () => {
+    const projection = await BucketCountProjection.open(join(projRoot, 'ii'))
+    const engine = new ReprojectionEngine({ source: sourceFor(brain), batchSize: 4 })
+    engine.register(projection)
+    const head = brain.scanFacts()!.headGeneration
+
+    const inFlight = engine.advance(projection.family, { budgetMs: 60_000 })
+    // The read hook: foreground door traffic announces itself, then reads —
+    // both interleave with the running fold on the same event loop.
+    engine.doorSignal.bump()
+    const found = await brain.find({ query: 'record filed in bucket', limit: 3 })
+    const got = await brain.get(ids[0])
+    const result = await inFlight
+
+    // The doors answered mid-fold.
+    expect(found.length).toBeGreaterThan(0)
+    expect(got).toBeTruthy()
+    const gotMeta = got!.metadata as Record<string, unknown> | undefined
+    expect((gotMeta?.bucket ?? (gotMeta?.metadata as Record<string, unknown>)?.bucket)).toBe('b0')
+
+    // THE PREEMPTION PIN — mechanism, not wall-clock: the bump landed before
+    // the first installment boundary, so the advance yielded after exactly
+    // one batch (≤ batchSize facts), far short of the head.
+    expect(result.status).toBe('preempted')
+    expect(result.applied).toBeGreaterThan(0)
+    expect(result.applied).toBeLessThanOrEqual(4)
+    expect(projection.appliedGenerations.length).toBe(result.applied)
+    expect(projection.watermark()).not.toBeNull()
+    expect(projection.watermark()!).toBeLessThan(head)
+
+    // Resuming folds the remainder; nothing was lost to the preemption.
+    const resumed = await engine.advance(projection.family, { budgetMs: 60_000 })
+    expect(resumed.status).toBe('caught-up')
+    expect(projection.counts()).toEqual(GROUND_TRUTH)
+  })
+
+  it('(iii) crash mid-fold: reopen and re-advance resumes from the stamp, never refolds from zero', async () => {
+    const projDir = join(projRoot, 'iii')
+    const before = await BucketCountProjection.open(projDir)
+    const engine1 = new ReprojectionEngine({ source: sourceFor(brain), batchSize: 4 })
+    engine1.register(before)
+
+    // A zero budget folds exactly one guaranteed batch, then stops.
+    const partial = await engine1.advance(before.family, { budgetMs: 0 })
+    expect(partial.status).toBe('budget-exhausted')
+    const stamped = before.watermark()
+    expect(stamped).not.toBeNull()
+    expect(stamped!).toBeGreaterThan(0)
+
+    // CRASH: abandon the engine and adapter mid-fold; reopen the brain cold.
+    await brain.close()
+    brain = await openBrain(brainDir)
+
+    const after = await BucketCountProjection.open(projDir)
+    expect(after.watermark()).toBe(stamped) // the stamp survived the crash
+
+    const engine2 = new ReprojectionEngine({ source: sourceFor(brain), batchSize: 4 })
+    engine2.register(after)
+    const resumed = await engine2.advance(after.family, { budgetMs: 60_000 })
+    expect(resumed.status).toBe('caught-up')
+
+    // NEVER REFOLDS FROM ZERO: every generation the resumed instance applied
+    // sits strictly above the crash stamp.
+    expect(after.appliedGenerations.length).toBeGreaterThan(0)
+    expect(Math.min(...after.appliedGenerations)).toBeGreaterThan(stamped!)
+    // And the combined state — durable prefix plus resumed fold — is exact.
+    expect(after.counts()).toEqual(GROUND_TRUTH)
+  })
+})