feat(log): v2 is the LIVE write format — envelope records with minted ints, genesis, sector seals; v1 readable forever
The cutover: new tail segments write format v2 (per-record [type, version, cipherFlag, keyId] envelope; noun/verb after-images carry dense ints MINTED AT APPEND from the id mapper — a rebuilt mapper reproduces assignments exactly; log.genesis opens every new log with the id-space width + a minted brain id; sync() seals to the header-declared sector boundary with reader-invisible pad frames). Existing v1 segments are never rewritten — per-segment decoder dispatch reads both formats and v2 facts map to the exact CommitFact shape all consumers already read. Cutover on a live v1 log: an empty v1 tail re-heads in place; a non-empty one is sealed by rotation, byte-identical. Records reserve the encryption fields (cipherFlag 0 / keyId nil are the only legal values; anything else refuses typed naming the needed newer reader) — crypto-ready with no future bump on the compat surface. Empty-records facts are legal (an all-deduped batch is a real generation — v1 semantics preserved; the refusal there tore a column-store flush mid-commit in the full suite, the consistency guard caught it loudly, and the root is fixed). Golden byte vectors pinned for the second (native) reader implementation. Pins: cutover 5/5 · codec 54 · kill-matrix stays 11/11.
This commit is contained in:
parent
73eb88d481
commit
26c6025158
6 changed files with 1372 additions and 135 deletions
389
tests/integration/fact-log-v2-cutover.test.ts
Normal file
389
tests/integration/fact-log-v2-cutover.test.ts
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
/**
|
||||
* @module tests/integration/fact-log-v2-cutover
|
||||
* @description The fact log's LIVE WRITE FORMAT cutover to v2, end-to-end
|
||||
* through real brains: (a) a NEW brain's tail segment carries a v2 header
|
||||
* (formatVersion 2, sealSize 4096), opens with the log.genesis record
|
||||
* (id-space width 64 + the manifest-persisted brainId), and scanFacts yields
|
||||
* the same CommitFact shape a v1 brain would — reconstruction included,
|
||||
* proven by digest-equality against canonical after a reopen; (b) MIXED
|
||||
* logs: an existing v1 segment stays readable forever beside a v2 tail
|
||||
* (cutover-by-rotation; the v1 segment is never rewritten); (c) MINT:
|
||||
* after-image records carry the metadata index id mapper's exact int
|
||||
* assignments (white-box compare); (d) SEALS: every flush leaves the tail
|
||||
* sector-aligned, and pads are invisible to scans; (e) REPLAY: the
|
||||
* log-authority recovery path resurrects an acked write from a v2 tail
|
||||
* after a crash-style abandon.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
import {
|
||||
parseSegmentHeader,
|
||||
decodeGroupV2,
|
||||
SEGMENT_HEADER_BYTES,
|
||||
FACT_LOG_FORMAT_V1,
|
||||
FACT_LOG_FORMAT_V2,
|
||||
type LogGenesisRecord,
|
||||
type NounAfterImageRecord
|
||||
} from '../../src/db/factLogFormat.js'
|
||||
import type { CommitFact, FactIntMinter, FactLog } from '../../src/db/factLog.js'
|
||||
import {
|
||||
makeTempDir,
|
||||
openBrain,
|
||||
storeOf,
|
||||
abandonAsCrashed,
|
||||
factGenerations,
|
||||
vec,
|
||||
uid
|
||||
} from '../helpers/durabilityKillMatrix.js'
|
||||
|
||||
/** The VFS root — created at init by a baseline (generation-less) write. */
|
||||
const VFS_ROOT = '00000000-0000-0000-0000-000000000000'
|
||||
const FACTS_DIR = ['_generations', 'facts'] as const
|
||||
const MANIFEST_PATH = '_generations/facts/manifest.json'
|
||||
|
||||
/** White-box internals this suite instruments. */
|
||||
type BrainInternals = {
|
||||
storage: {
|
||||
readRawObject(p: string): Promise<unknown | null>
|
||||
readNounRaw(id: string): Promise<{ metadata: unknown | null; vector: unknown | null }>
|
||||
}
|
||||
metadataIndex: {
|
||||
getIdMapper(): { getInt(uuid: string): number | undefined }
|
||||
}
|
||||
}
|
||||
const internals = (brain: Brainy): BrainInternals => brain as unknown as BrainInternals
|
||||
|
||||
/** The facts manifest as stored (additive brainId included). */
|
||||
interface StoredFactsManifest {
|
||||
segments: Array<{ file: string }>
|
||||
tailSegment: string | null
|
||||
brainId?: string
|
||||
}
|
||||
|
||||
async function readManifest(brain: Brainy): Promise<StoredFactsManifest> {
|
||||
const manifest = (await internals(brain).storage.readRawObject(
|
||||
MANIFEST_PATH
|
||||
)) as StoredFactsManifest | null
|
||||
expect(manifest, 'the facts manifest exists').toBeTruthy()
|
||||
return manifest!
|
||||
}
|
||||
|
||||
/** Raw on-disk bytes of one fact segment file. */
|
||||
function segmentBytes(dir: string, file: string): Uint8Array {
|
||||
return new Uint8Array(fs.readFileSync(path.join(dir, ...FACTS_DIR, file)))
|
||||
}
|
||||
|
||||
async function allFacts(brain: Brainy): Promise<CommitFact[]> {
|
||||
const scan = (brain as unknown as { scanFacts(): { batches(): AsyncGenerator<{ facts: CommitFact[] }> } | null }).scanFacts()
|
||||
expect(scan, 'this storage hosts a fact log').not.toBeNull()
|
||||
const facts: CommitFact[] = []
|
||||
for await (const batch of scan!.batches()) facts.push(...batch.facts)
|
||||
return facts
|
||||
}
|
||||
|
||||
/** The live FactLog instance (white-box: the minter strip in scenario b). */
|
||||
function factLogOf(brain: Brainy): FactLog & { intMinter: FactIntMinter | null } {
|
||||
const log = storeOf(brain).getFactLog()
|
||||
expect(log, 'filesystem storage hosts a fact log').not.toBeNull()
|
||||
return log as FactLog & { intMinter: FactIntMinter | null }
|
||||
}
|
||||
|
||||
describe('fact log v2 cutover — live writes land in the v2 segment format', () => {
|
||||
const dirs: string[] = []
|
||||
const brains: Brainy[] = []
|
||||
|
||||
const trackDir = (): string => {
|
||||
const dir = makeTempDir()
|
||||
dirs.push(dir)
|
||||
return dir
|
||||
}
|
||||
const track = (brain: Brainy): Brainy => {
|
||||
brains.push(brain)
|
||||
return brain
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) {
|
||||
await (b as unknown as { close?: () => Promise<void> }).close?.().catch(() => {})
|
||||
}
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('(a) NEW BRAIN: v2 tail header, genesis-first, and scanFacts parity with canonical across a reopen', async () => {
|
||||
const dir = trackDir()
|
||||
const brain = track(await openBrain(dir))
|
||||
const idA = uid('v2-new-a')
|
||||
const idB = uid('v2-new-b')
|
||||
await brain.add({ id: idA, data: 'alpha', type: NounType.Document, vector: vec(1), metadata: { n: 1 } })
|
||||
await brain.add({ id: idB, data: 'beta', type: NounType.Document, vector: vec(2), metadata: { n: 2 } })
|
||||
await brain.flush()
|
||||
|
||||
// The tail segment's raw header bytes: formatVersion 2, sealSize 4096.
|
||||
const manifest = await readManifest(brain)
|
||||
expect(manifest.tailSegment).toBeTruthy()
|
||||
expect(manifest.brainId, 'the brain id was minted into the manifest').toBeTruthy()
|
||||
const bytes = segmentBytes(dir, manifest.tailSegment!)
|
||||
const header = parseSegmentHeader(bytes.subarray(0, SEGMENT_HEADER_BYTES))
|
||||
expect(header.formatVersion).toBe(FACT_LOG_FORMAT_V2)
|
||||
expect(header.sealSize).toBe(4096)
|
||||
|
||||
// Genesis is the FIRST record of the FIRST fact — and appears exactly once.
|
||||
const group = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES), { expectedIdSpaceWidth: 64 })
|
||||
expect(group.facts.length).toBeGreaterThanOrEqual(2)
|
||||
const firstRecord = group.facts[0].records[0]
|
||||
expect(firstRecord.type).toBe('log.genesis')
|
||||
const genesis = firstRecord as LogGenesisRecord
|
||||
expect(genesis.idSpaceWidth).toBe(64)
|
||||
expect(genesis.brainId).toBe(manifest.brainId)
|
||||
const genesisCount = group.facts
|
||||
.flatMap((f) => f.records)
|
||||
.filter((r) => r.type === 'log.genesis').length
|
||||
expect(genesisCount).toBe(1)
|
||||
|
||||
// Shape parity + reconstruction fidelity: REOPEN (so the tail decodes
|
||||
// from disk, not from the in-session originals) and compare each add's
|
||||
// CommitFact op against canonical byte truth — metadata leg (bigint
|
||||
// timestamps normalized back to numbers) AND the reconstructed vector
|
||||
// wrapper must equal what readNounRaw returns, exactly as a v1 log's
|
||||
// byte-faithful capture would.
|
||||
await (brain as unknown as { close: () => Promise<void> }).close()
|
||||
brains.splice(brains.indexOf(brain), 1)
|
||||
const reopened = track(await openBrain(dir))
|
||||
const facts = await allFacts(reopened)
|
||||
const gens = facts.map((f) => f.generation)
|
||||
expect([...gens].sort((a, b) => a - b)).toEqual(gens)
|
||||
expect(new Set(gens).size).toBe(gens.length)
|
||||
|
||||
const logGens = new Set(
|
||||
((await (reopened as unknown as { transactionLog(): Promise<Array<{ generation: number }>> }).transactionLog()) ?? []).map(
|
||||
(e) => e.generation
|
||||
)
|
||||
)
|
||||
for (const g of gens) expect(logGens.has(g), `generation ${g} is a real commit`).toBe(true)
|
||||
|
||||
for (const id of [idA, idB]) {
|
||||
const fact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null))
|
||||
expect(fact, `the add fact for ${id} survives the reopen`).toBeDefined()
|
||||
const op = fact!.ops.find((o) => o.id === id)!
|
||||
expect(op.kind).toBe('noun')
|
||||
const canonical = await internals(reopened).storage.readNounRaw(id)
|
||||
expect(op.record!.metadata).toStrictEqual(canonical.metadata)
|
||||
expect(op.record!.vector).toStrictEqual(canonical.vector)
|
||||
}
|
||||
})
|
||||
|
||||
it('(b) MIXED LOG: an existing v1 segment stays readable forever beside the v2 tail (cutover by rotation, v1 bytes untouched)', async () => {
|
||||
// ROUTE: a REAL v1 segment is written by the v1 writer itself — the live
|
||||
// FactLog with its minter stripped (the exact pre-cutover code path,
|
||||
// still shipped for minter-less configurations) — then the minter is
|
||||
// restored mid-session and the next append performs the cutover
|
||||
// rotation. Stronger than hand-crafted bytes: both formats come from
|
||||
// their real writers, on one log.
|
||||
const dir = trackDir()
|
||||
const brain = track(await openBrain(dir))
|
||||
const log = factLogOf(brain)
|
||||
const minter = log.intMinter
|
||||
expect(minter, 'the brain wired the int minter at init').toBeTruthy()
|
||||
|
||||
log.intMinter = null // the pre-cutover writer
|
||||
const idOld1 = uid('v1-old-1')
|
||||
const idOld2 = uid('v1-old-2')
|
||||
await brain.add({ id: idOld1, data: 'old one', type: NounType.Document, vector: vec(3), metadata: { era: 'v1' } })
|
||||
await brain.add({ id: idOld2, data: 'old two', type: NounType.Document, vector: vec(4), metadata: { era: 'v1' } })
|
||||
await brain.flush()
|
||||
|
||||
const before = await readManifest(brain)
|
||||
expect(before.segments).toHaveLength(0)
|
||||
const v1TailFile = before.tailSegment!
|
||||
const v1Bytes = segmentBytes(dir, v1TailFile)
|
||||
expect(parseSegmentHeader(v1Bytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe(
|
||||
FACT_LOG_FORMAT_V1
|
||||
)
|
||||
|
||||
log.intMinter = minter // the cutover lands mid-session
|
||||
const idNew = uid('v2-new')
|
||||
await brain.add({ id: idNew, data: 'new era', type: NounType.Document, vector: vec(5), metadata: { era: 'v2' } })
|
||||
await brain.flush()
|
||||
|
||||
// The v1 tail was SEALED (bytes untouched), the new tail is v2.
|
||||
const after = await readManifest(brain)
|
||||
expect(after.segments.map((s) => s.file)).toContain(v1TailFile)
|
||||
expect(after.tailSegment).not.toBe(v1TailFile)
|
||||
const sealedBytes = segmentBytes(dir, v1TailFile)
|
||||
expect(parseSegmentHeader(sealedBytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe(
|
||||
FACT_LOG_FORMAT_V1
|
||||
)
|
||||
expect(
|
||||
Buffer.compare(Buffer.from(sealedBytes), Buffer.from(v1Bytes)),
|
||||
'the sealed v1 segment is byte-identical — never rewritten'
|
||||
).toBe(0)
|
||||
const tailBytes = segmentBytes(dir, after.tailSegment!)
|
||||
expect(parseSegmentHeader(tailBytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe(
|
||||
FACT_LOG_FORMAT_V2
|
||||
)
|
||||
// NOT a brand-new log: no genesis on a rotated-in v2 tail.
|
||||
const tailGroup = decodeGroupV2(tailBytes.subarray(SEGMENT_HEADER_BYTES), {
|
||||
expectedIdSpaceWidth: 64
|
||||
})
|
||||
expect(
|
||||
tailGroup.facts.flatMap((f) => f.records).some((r) => r.type === 'log.genesis')
|
||||
).toBe(false)
|
||||
|
||||
// One scan spans both formats, shape-identically, in generation order.
|
||||
const liveFacts = await allFacts(brain)
|
||||
const liveGens = liveFacts.map((f) => f.generation)
|
||||
expect([...liveGens].sort((a, b) => a - b)).toEqual(liveGens)
|
||||
for (const id of [idOld1, idOld2, idNew]) {
|
||||
const fact = liveFacts.find((f) => f.ops.some((op) => op.id === id))
|
||||
expect(fact, `fact for ${id} is scannable`).toBeDefined()
|
||||
const op = fact!.ops.find((o) => o.id === id)!
|
||||
expect(op.kind).toBe('noun')
|
||||
expect(op.record).not.toBeNull()
|
||||
}
|
||||
|
||||
// The MIXED log survives a reopen and keeps appending (v2 tail).
|
||||
await (brain as unknown as { close: () => Promise<void> }).close()
|
||||
brains.splice(brains.indexOf(brain), 1)
|
||||
const reopened = track(await openBrain(dir))
|
||||
const reFacts = await allFacts(reopened)
|
||||
expect(reFacts.map((f) => f.generation)).toEqual(liveGens)
|
||||
// The v1 fact still reads exactly as the v1 decoder always read it.
|
||||
// (Not compared byte-strict against canonical: the v1 CAPTURE has a
|
||||
// known pre-existing wart — write-cache-warm objects carry
|
||||
// undefined-valued engine keys that msgpack preserves as nil while the
|
||||
// durable JSON drops them. v1 bytes are frozen; the v2 encoder
|
||||
// sanitizes to durable truth instead — pinned in scenario (a).)
|
||||
const oldOp = reFacts
|
||||
.find((f) => f.ops.some((op) => op.id === idOld1))!
|
||||
.ops.find((o) => o.id === idOld1)!
|
||||
const canonicalOld = await internals(reopened).storage.readNounRaw(idOld1)
|
||||
const oldMeta = oldOp.record!.metadata as Record<string, unknown>
|
||||
expect(oldMeta.noun).toBe('document')
|
||||
expect((oldMeta.metadata as Record<string, unknown>).era).toBe('v1')
|
||||
const oldWrapper = oldOp.record!.vector as { id: string; vector: number[] }
|
||||
const canonicalWrapper = canonicalOld.vector as { id: string; vector: number[] }
|
||||
expect(oldWrapper.id).toBe(idOld1)
|
||||
expect(oldWrapper.vector).toStrictEqual(canonicalWrapper.vector)
|
||||
await reopened.add({ id: uid('post-reopen'), data: 'still writing', type: NounType.Document, vector: vec(6), metadata: {} })
|
||||
expect((await factGenerations(reopened)).length).toBe(liveGens.length + 1)
|
||||
})
|
||||
|
||||
it('(c) MINT-AT-APPEND: after-image records carry the id mapper\'s EXACT int assignments — distinct, nonzero, reproducible', async () => {
|
||||
const dir = trackDir()
|
||||
const brain = track(await openBrain(dir))
|
||||
const idA = uid('mint-a')
|
||||
const idB = uid('mint-b')
|
||||
await brain.add({ id: idA, data: 'mint one', type: NounType.Document, vector: vec(7), metadata: { m: 1 } })
|
||||
await brain.add({ id: idB, data: 'mint two', type: NounType.Document, vector: vec(8), metadata: { m: 2 } })
|
||||
await brain.flush()
|
||||
|
||||
const manifest = await readManifest(brain)
|
||||
const bytes = segmentBytes(dir, manifest.tailSegment!)
|
||||
const group = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES), { expectedIdSpaceWidth: 64 })
|
||||
const afterImages = new Map<string, NounAfterImageRecord>()
|
||||
for (const fact of group.facts) {
|
||||
for (const record of fact.records) {
|
||||
if (record.type === 'noun.afterImage') afterImages.set(record.id, record)
|
||||
}
|
||||
}
|
||||
const recA = afterImages.get(idA)
|
||||
const recB = afterImages.get(idB)
|
||||
expect(recA, 'idA has a decoded after-image').toBeDefined()
|
||||
expect(recB, 'idB has a decoded after-image').toBeDefined()
|
||||
expect(recA!.entityInt).toBeGreaterThan(0n)
|
||||
expect(recB!.entityInt).toBeGreaterThan(0n)
|
||||
expect(recA!.entityInt).not.toBe(recB!.entityInt)
|
||||
|
||||
// White-box: the ints on the wire ARE the metadata index mapper's
|
||||
// assignments — the exact ints a mapper rebuild must reproduce.
|
||||
const mapper = internals(brain).metadataIndex.getIdMapper()
|
||||
expect(recA!.entityInt).toBe(BigInt(mapper.getInt(idA)!))
|
||||
expect(recB!.entityInt).toBe(BigInt(mapper.getInt(idB)!))
|
||||
})
|
||||
|
||||
it('(d) SEALS AT SYNC: every flush leaves the tail sector-aligned; pads are invisible to scans', async () => {
|
||||
const dir = trackDir()
|
||||
const brain = track(await openBrain(dir))
|
||||
await brain.add({ id: uid('seal-1'), data: 'one', type: NounType.Document, vector: vec(10), metadata: {} })
|
||||
await brain.flush()
|
||||
|
||||
const manifest = await readManifest(brain)
|
||||
const tailPath = path.join(dir, ...FACTS_DIR, manifest.tailSegment!)
|
||||
const sizeAfterFirstFlush = fs.statSync(tailPath).size
|
||||
expect(sizeAfterFirstFlush).toBeGreaterThan(0)
|
||||
expect(sizeAfterFirstFlush % 4096, 'tail is sector-aligned after flush').toBe(0)
|
||||
const countAfterFirstFlush = (await factGenerations(brain)).length
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await brain.add({ id: uid(`seal-more-${i}`), data: `more ${i}`, type: NounType.Document, vector: vec(11 + i), metadata: { i } })
|
||||
}
|
||||
await brain.flush()
|
||||
const sizeAfterSecondFlush = fs.statSync(tailPath).size
|
||||
expect(sizeAfterSecondFlush).toBeGreaterThan(sizeAfterFirstFlush)
|
||||
expect(sizeAfterSecondFlush % 4096, 'still aligned after more writes + flush').toBe(0)
|
||||
|
||||
// Pads count toward bytes, never toward facts.
|
||||
expect((await factGenerations(brain)).length).toBe(countAfterFirstFlush + 3)
|
||||
})
|
||||
|
||||
it('(e) REPLAY COMPAT: the log-authority recovery path resurrects an acked write from a v2 tail after a crash-style abandon', async () => {
|
||||
// The flip idiom from the log-authority suite: seed writes, baseline
|
||||
// backfill LAST (the init-time VFS root never got a fact), flush, then
|
||||
// the sanctioned guarded flip — the oracle goes green over an ALL-V2
|
||||
// log, which is itself the reproduction proof for the v2 record path.
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-v2-cutover-'))
|
||||
dirs.push(dir)
|
||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||
const open = async (): Promise<Brainy> => {
|
||||
const b = new Brainy({
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
requireSubtype: false,
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
await b.init()
|
||||
return track(b)
|
||||
}
|
||||
|
||||
const brain = await open()
|
||||
const kept = await brain.add({ data: 'alpha document', type: 'document', metadata: { n: 1 } })
|
||||
const removed = await brain.add({ data: 'beta document', type: 'document', metadata: { n: 2 } })
|
||||
await brain.update({ id: kept, metadata: { n: 10 } })
|
||||
await brain.remove(removed)
|
||||
const root = await brain.get(VFS_ROOT)
|
||||
expect(root, 'the VFS root exists').toBeTruthy()
|
||||
await brain.update({ id: VFS_ROOT, metadata: root!.metadata }) // baseline backfill — final write
|
||||
await brain.flush()
|
||||
|
||||
const report = await (brain as unknown as { adoptLogAuthority(): Promise<{ verdict: string }> }).adoptLogAuthority()
|
||||
expect(report.verdict, 'the oracle is green over a pure-v2 log').toBe('green')
|
||||
|
||||
// An at-ack write: its v2 fact is fsynced (sector-sealed) at ack.
|
||||
const survivor = await brain.add({
|
||||
data: 'survives power loss',
|
||||
type: 'document',
|
||||
metadata: { s: 1 }
|
||||
})
|
||||
|
||||
// Crash-style abandon: RAM state gone, no flush, no close.
|
||||
await abandonAsCrashed(brain)
|
||||
|
||||
// Reopen: open() finds the acked fact ABOVE the manifest watermark in
|
||||
// the v2 tail (peekFactsAbove → v2 decode) and REPLAYS it into
|
||||
// canonical — an acked write is never lost.
|
||||
const reopened = await open()
|
||||
expect(
|
||||
(reopened as unknown as { logAuthority(): { authority: string } }).logAuthority().authority
|
||||
).toBe('log')
|
||||
const resurrected = await reopened.get(survivor)
|
||||
expect(resurrected, 'the acked write survived the crash').toBeTruthy()
|
||||
expect((resurrected as { metadata?: { s?: number } }).metadata?.s).toBe(1)
|
||||
expect((await factGenerations(reopened)).length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
|
@ -41,6 +41,7 @@ type BrainInternals = {
|
|||
saveNoun(n: unknown): Promise<void>
|
||||
saveNounMetadata(id: string, m: Record<string, unknown>): Promise<void>
|
||||
getNounMetadata(id: string): Promise<Record<string, unknown> | null>
|
||||
writeNounRaw(id: string, r: { metadata: null; vector: null }): Promise<void>
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -219,28 +220,21 @@ describe('log authority — the switch, the oracle, the guarded flip', () => {
|
|||
})
|
||||
})
|
||||
|
||||
it('THE FLIP REFUSES ON RED: names the oracle verdict and the cure, writes nothing, changes nothing', async () => {
|
||||
it('THE FLIP REFUSES ON A LOG-AHEAD DIVERGENCE: the witness denies what the log claims — nothing written, nothing changed', async () => {
|
||||
// Contract update (adoptLogAuthority's baseline backfill): curable
|
||||
// divergences — pre-log records and witness drift — are re-committed
|
||||
// and the flip proceeds; ONLY log-AHEAD divergences (the log claims
|
||||
// state canonical denies) refuse, because no backfill can make the log
|
||||
// un-claim a live row. This test stages exactly that incurable shape.
|
||||
const { brain } = await openBrain()
|
||||
await seedWrites(brain)
|
||||
const { kept } = await seedWrites(brain)
|
||||
await backfillBaseline(brain)
|
||||
await brain.flush()
|
||||
|
||||
// Age the brain: one canonical record the log never saw.
|
||||
const legacyId = '00000000-0000-4000-8000-00000000a6ed'
|
||||
// The log says `kept` is live; its canonical record vanishes behind the
|
||||
// write path's back (log-live-canonical-absent — the witness wins).
|
||||
const storage = internals(brain).storage
|
||||
await storage.saveNoun({
|
||||
id: legacyId,
|
||||
vector: new Array(384).fill(0.01),
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
})
|
||||
await storage.saveNounMetadata(legacyId, {
|
||||
noun: 'document',
|
||||
confidence: 0.5,
|
||||
createdAt: 1700000000000,
|
||||
updatedAt: 1700000000000,
|
||||
_rev: 1
|
||||
})
|
||||
await storage.writeNounRaw(kept, { metadata: null, vector: null })
|
||||
|
||||
let error: Error | null = null
|
||||
try {
|
||||
|
|
@ -248,9 +242,9 @@ describe('log authority — the switch, the oracle, the guarded flip', () => {
|
|||
} catch (err) {
|
||||
error = err as Error
|
||||
}
|
||||
expect(error, 'the flip rejects on a red oracle').not.toBeNull()
|
||||
expect(error!.message).toMatch(/oracle is RED/)
|
||||
expect(error!.message).toMatch(/baseline backfill/)
|
||||
expect(error, 'the flip rejects on a log-ahead divergence').not.toBeNull()
|
||||
expect(error!.message).toMatch(/witness denies/)
|
||||
expect(error!.message).toMatch(/log-live-canonical-absent/)
|
||||
|
||||
// Nothing changed: authority still tree, no artifact, deferred durability.
|
||||
expect(brain.logAuthority().authority).toBe('tree')
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@
|
|||
* @description Fact-log format v2 (record envelope + sector seals) pinned at
|
||||
* the byte level: every record type round-trips field-exact (bigint ints,
|
||||
* bin16 uuids, float-exact vectors), headers read v1 AND v2, unknown record
|
||||
* types/versions refuse loudly with the typed error, genesis width mismatches
|
||||
* types/versions refuse loudly with the typed error, the reserved crypto
|
||||
* envelope (cipherFlag/keyId — plaintext-only this release) refuses anything
|
||||
* nonzero/non-nil with the same typed error, genesis width mismatches
|
||||
* refuse naming both widths, sealed groups align to the sector size with
|
||||
* invisible pads, vector refs are writer-enforced single-hop, and torn tails
|
||||
* truncate to the intact prefix at EVERY byte offset. This module is the
|
||||
|
|
@ -11,7 +13,7 @@
|
|||
* vectors here are frozen; a change that breaks them is a format change.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { encode } from '@msgpack/msgpack'
|
||||
import { encode, decode } from '@msgpack/msgpack'
|
||||
import {
|
||||
encodeFactV2,
|
||||
decodeFact,
|
||||
|
|
@ -20,10 +22,13 @@ import {
|
|||
parseSegmentHeader,
|
||||
sealGroup,
|
||||
framePayload,
|
||||
encodePadFrame,
|
||||
minPadFrameBytes,
|
||||
UnknownLogRecordError,
|
||||
GenesisWidthMismatchError,
|
||||
LOG_RECORD_TYPES,
|
||||
LOG_RECORD_VERSION,
|
||||
LOG_RECORD_CIPHER_PLAINTEXT,
|
||||
FACT_LOG_FORMAT_V1,
|
||||
FACT_LOG_FORMAT_V2,
|
||||
SEGMENT_HEADER_BYTES,
|
||||
|
|
@ -254,7 +259,7 @@ describe('fact-log format v2 — golden byte vectors (frozen contract)', () => {
|
|||
records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }]
|
||||
})
|
||||
expect(hex(frame)).toBe(
|
||||
'2b000000c19ad9ff95cf0000000000000003cf0000018bcfe5687b91930201' +
|
||||
'2d00000048e4d43695cf0000000000000003cf0000018bcfe5687b9195020100c0' +
|
||||
'c41000000000000040008000000000000042c0c0'
|
||||
)
|
||||
})
|
||||
|
|
@ -370,11 +375,51 @@ describe('fact-log format v2 — decoder law (typed refusals, never skip)', () =
|
|||
})
|
||||
|
||||
it('a fact mixing known and unknown records still refuses (no partial reads)', () => {
|
||||
const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, uuidBytes(UUID(1))]
|
||||
const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))]
|
||||
const payload = encode([1, 1, [known, [200, 1]], null, null])
|
||||
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
|
||||
})
|
||||
|
||||
it('a nonzero cipherFlag refuses with the typed error — encrypted records need a newer reader', () => {
|
||||
const payload = encode(
|
||||
[1, 1, [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 1, null, uuidBytes(UUID(1))]], null, null]
|
||||
)
|
||||
try {
|
||||
decodeFact(payload, 2)
|
||||
expect.unreachable('decode must throw')
|
||||
} catch (error) {
|
||||
const typed = error as UnknownLogRecordError
|
||||
expect(typed).toBeInstanceOf(UnknownLogRecordError)
|
||||
expect(typed.recordType).toBe(LOG_RECORD_TYPES.NOUN_TOMBSTONE)
|
||||
expect(typed.recordVersion).toBe(1)
|
||||
expect(typed.message).toMatch(/cipherFlag 1/)
|
||||
expect(typed.message).toMatch(/encrypted records need a newer reader/)
|
||||
}
|
||||
})
|
||||
|
||||
it('a non-nil keyId refuses the same way, even with cipherFlag 0', () => {
|
||||
const payload = encode(
|
||||
[
|
||||
1,
|
||||
1,
|
||||
[[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, uuidBytes(UUID(9)), uuidBytes(UUID(1))]],
|
||||
null,
|
||||
null
|
||||
]
|
||||
)
|
||||
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
|
||||
expect(() => decodeFact(payload, 2)).toThrow(/encrypted records need a newer reader/)
|
||||
})
|
||||
|
||||
it('the encoder always writes the plaintext envelope: cipherFlag 0, keyId nil', () => {
|
||||
const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })))
|
||||
const raw = decode(payload) as unknown[]
|
||||
const record = (raw[2] as unknown[][])[0]
|
||||
expect(record[2]).toBe(LOG_RECORD_CIPHER_PLAINTEXT)
|
||||
expect(record[3]).toBeNull()
|
||||
expect(LOG_RECORD_CIPHER_PLAINTEXT).toBe(0)
|
||||
})
|
||||
|
||||
it('an unknown segment format version has no decode path', () => {
|
||||
const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })))
|
||||
expect(() => decodeFact(payload, 3)).toThrow(/reads 1 and 2/)
|
||||
|
|
@ -424,8 +469,8 @@ describe('fact-log format v2 — log.genesis width law', () => {
|
|||
1,
|
||||
1,
|
||||
[
|
||||
[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, uuidBytes(UUID(1))],
|
||||
[LOG_RECORD_TYPES.LOG_GENESIS, 1, 64, uuidBytes(UUID(9)), 1]
|
||||
[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))],
|
||||
[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 64, uuidBytes(UUID(9)), 1]
|
||||
],
|
||||
null,
|
||||
null
|
||||
|
|
@ -434,7 +479,9 @@ describe('fact-log format v2 — log.genesis width law', () => {
|
|||
})
|
||||
|
||||
it('an invalid genesis width on the wire is malformed, not a mismatch', () => {
|
||||
const crafted = encode([1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 48, uuidBytes(UUID(9)), 1]], null, null])
|
||||
const crafted = encode(
|
||||
[1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 48, uuidBytes(UUID(9)), 1]], null, null]
|
||||
)
|
||||
expect(() => decodeFact(crafted, 2)).toThrow(/32 or 64/)
|
||||
})
|
||||
})
|
||||
|
|
@ -504,7 +551,7 @@ describe('fact-log format v2 — vector legs (single-hop law)', () => {
|
|||
})
|
||||
expect(() => encodeFactV2(bad)).toThrow(/INLINE/)
|
||||
const craftedRef = encode(
|
||||
[1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, uuidBytes(UUID(7)), ['ref', 5]]], null, null]
|
||||
[1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, 0, null, uuidBytes(UUID(7)), ['ref', 5]]], null, null]
|
||||
)
|
||||
expect(() => decodeFact(craftedRef, 2)).toThrow(/INLINE/)
|
||||
})
|
||||
|
|
@ -571,16 +618,30 @@ describe('fact-log format v2 — sector seals', () => {
|
|||
timestamp: 1_700_000_000_123,
|
||||
records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }]
|
||||
})
|
||||
const sealed = sealGroup([tomb], 64) // 51 bytes → gap 13 → overshoot → 77-byte pad
|
||||
const sealed = sealGroup([tomb], 64) // 53 bytes → gap 11 → overshoot → 75-byte pad
|
||||
expect(sealed.length).toBe(128)
|
||||
expect(hex(sealed.subarray(tomb.length))).toBe(
|
||||
// frame prefix + [0, 0, [[0, 1, bin8(42 zero bytes)]], nil, nil]
|
||||
'450000009463044d95cf0000000000000000cf000000000000000091930001c42a' +
|
||||
'0'.repeat(84) +
|
||||
// frame prefix + [0, 0, [[0, 1, bin8(40 zero bytes)]], nil, nil]
|
||||
'4300000088b4c8fa95cf0000000000000000cf000000000000000091930001c428' +
|
||||
'0'.repeat(80) +
|
||||
'c0c0'
|
||||
)
|
||||
})
|
||||
|
||||
it('encodePadFrame builds exact-size pads for streaming writers; refuses sub-minimum sizes', () => {
|
||||
// Pads are envelope-exempt (skipped wholesale), so the smallest pad frame
|
||||
// is byte-stable across the crypto-envelope change.
|
||||
expect(minPadFrameBytes()).toBe(33)
|
||||
for (const size of [minPadFrameBytes(), 64, 4096]) {
|
||||
const pad = encodePadFrame(size)
|
||||
expect(pad.length).toBe(size)
|
||||
const { facts: decoded, validBytes } = decodeGroupV2(pad)
|
||||
expect(decoded).toEqual([]) // invisible to readers
|
||||
expect(validBytes).toBe(size)
|
||||
}
|
||||
expect(() => encodePadFrame(minPadFrameBytes() - 1)).toThrow(/at least/)
|
||||
})
|
||||
|
||||
it('sealGroup refuses garbage: empty groups, malformed frames, bad seal sizes', () => {
|
||||
expect(() => sealGroup([], 4096)).toThrow(/at least one frame/)
|
||||
expect(() => sealGroup([new Uint8Array([1, 2, 3])], 4096)).toThrow(/not a well-formed frame/)
|
||||
|
|
@ -620,10 +681,12 @@ describe('fact-log format v2 — torn-tail discipline', () => {
|
|||
describe('fact-log format v2 — writer refusals (loud, never silent)', () => {
|
||||
const tombstone = (g: number): CommitFactV2 => factOf(g, { type: 'noun.tombstone', id: UUID(g) })
|
||||
|
||||
it('refuses empty records, generation 0, and a second batch.meta', () => {
|
||||
expect(() => encodeFactV2({ generation: 1, timestamp: 1, records: [] })).toThrow(
|
||||
/at least one record/
|
||||
)
|
||||
it('accepts empty records (an all-deduped batch is a real generation); refuses generation 0 and a second batch.meta', () => {
|
||||
// Contract change with the live cutover: v1 always encoded op-less
|
||||
// commits (a batch whose relates dedupe away still mints a generation);
|
||||
// v2 must not fork commit semantics — empty records round-trip.
|
||||
const empty = decodeFact(framePayload(encodeFactV2({ generation: 1, timestamp: 1, records: [] })), 2)
|
||||
expect(empty.records).toEqual([])
|
||||
expect(() => encodeFactV2({ ...tombstone(1), generation: 0 })).toThrow(/positive integer/)
|
||||
expect(() =>
|
||||
encodeFactV2({
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue