From 26c6025158cdf70683ccd625cb395f2dda11f9b1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 10 Aug 2026 10:55:11 -0700 Subject: [PATCH] =?UTF-8?q?feat(log):=20v2=20is=20the=20LIVE=20write=20for?= =?UTF-8?q?mat=20=E2=80=94=20envelope=20records=20with=20minted=20ints,=20?= =?UTF-8?q?genesis,=20sector=20seals;=20v1=20readable=20forever?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cutover: new tail segments write format v2 (per-record [type, version, cipherFlag, keyId] envelope; noun/verb after-images carry dense ints MINTED AT APPEND from the id mapper — a rebuilt mapper reproduces assignments exactly; log.genesis opens every new log with the id-space width + a minted brain id; sync() seals to the header-declared sector boundary with reader-invisible pad frames). Existing v1 segments are never rewritten — per-segment decoder dispatch reads both formats and v2 facts map to the exact CommitFact shape all consumers already read. Cutover on a live v1 log: an empty v1 tail re-heads in place; a non-empty one is sealed by rotation, byte-identical. Records reserve the encryption fields (cipherFlag 0 / keyId nil are the only legal values; anything else refuses typed naming the needed newer reader) — crypto-ready with no future bump on the compat surface. Empty-records facts are legal (an all-deduped batch is a real generation — v1 semantics preserved; the refusal there tore a column-store flush mid-commit in the full suite, the consistency guard caught it loudly, and the root is fixed). Golden byte vectors pinned for the second (native) reader implementation. Pins: cutover 5/5 · codec 54 · kill-matrix stays 11/11. --- src/db/factLog.ts | 753 +++++++++++++++++- src/db/factLogFormat.ts | 211 +++-- src/db/generationStore.ts | 25 +- tests/integration/fact-log-v2-cutover.test.ts | 389 +++++++++ tests/integration/log-authority.test.ts | 34 +- tests/unit/db/factLogFormat.test.ts | 95 ++- 6 files changed, 1372 insertions(+), 135 deletions(-) create mode 100644 tests/integration/fact-log-v2-cutover.test.ts diff --git a/src/db/factLog.ts b/src/db/factLog.ts index 19bbb10e..c005d74e 100644 --- a/src/db/factLog.ts +++ b/src/db/factLog.ts @@ -41,10 +41,57 @@ * terminal-readable) is the single source of truth for the segment SET; * rotation flips it atomically (write-new → fsync → rename) BEFORE the new * tail's first byte exists, so no segment file is ever unaccounted for. + * + * ## Mixed-version logs (the v2 live-write cutover) + * + * The segment header's `formatVersion` selects the decoder PER SEGMENT: + * v1 segments (ops-shaped facts, the format above) stay readable forever and + * are NEVER rewritten; a NEW tail segment writes the v2 format + * (`src/db/factLogFormat.ts` — record envelope, minted dense ints, genesis, + * sector seals) whenever the int minter is installed ({@link FactLog.setIntMinter} — + * the brain wires it from the metadata index's id mapper right after init). + * A bare `FactLog` with no minter keeps writing v1 (there is no authority + * that could reproduce int assignments, and 0 is never written). Cutover + * mechanics on an existing v1 log: an EMPTY v1 tail is re-headed to v2 in + * place; a non-empty v1 tail is sealed by an immediate rotation and the new + * tail is v2. Decoded v2 facts map back to the SAME {@link CommitFact} shape + * v1 consumers read (noun/verb ops with `{metadata, vector} | null` records) — + * the vector wrapper object is reconstructed from the record's metadata leg + * through the reserved-field hydration law (see `commitFactFromV2`). + * + * V2 tails additionally: write the `log.genesis` record (id-space width 64 + + * the brain id, minted once into the manifest's additive `brainId` field) as + * the first record of the FIRST fact of a brand-new log, and seal every + * `sync()` to the header-declared sector size with pad frames that are + * invisible to readers (torn-page defense at group-commit boundaries). */ import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack' import { crc32c } from '../utils/crc32c.js' import { prodLog } from '../utils/logger.js' +import { + FACT_LOG_FORMAT_V1, + FACT_LOG_FORMAT_V2, + DEFAULT_SEAL_SIZE, + parseSegmentHeader, + encodeSegmentHeaderV2, + encodeFactV2, + decodeFact as decodeFormatFact, + decodeGroupV2, + encodePadFrame, + minPadFrameBytes, + type CommitFactV2, + type LogRecord, + type EmbedPendingRecord, + type EmbedLandedRecord, + type BlobManifestRecord, + type BootstrapBaselineRecord, + type ProjectionNoteRecord +} from './factLogFormat.js' +import { + splitNounMetadataRecord +} from '../types/reservedFields.js' +import { NounType } from '../types/graphTypes.js' +import { v4 as uuidv4 } from '../universal/uuid.js' // Swappable msgpack implementation — defaults to the JS codec; a native // provider (registered via the plugin registry's 'msgpack' key) may replace @@ -65,7 +112,12 @@ export function setFactCodec(impl: { export const FACTS_PREFIX = '_generations/facts' /** The facts manifest path (JSON). */ export const FACTS_MANIFEST_PATH = `${FACTS_PREFIX}/manifest.json` -/** Current segment format version (header field; additive-only within a major). */ +/** + * The v1 segment format version — the MANIFEST's formatVersion gate and the + * header value of v1 (minter-less) tails. NOT the live-write ceiling: new + * tails write `FACT_LOG_FORMAT_V2` (src/db/factLogFormat.ts) whenever the + * int minter is installed; both versions are read forever, per segment. + */ export const FACTS_FORMAT_VERSION = 1 /** Rotation threshold: seal the tail segment once it exceeds this many bytes. */ const SEGMENT_ROTATE_BYTES = 8 * 1024 * 1024 @@ -83,6 +135,29 @@ export interface FactOp { record: { metadata: unknown | null; vector: unknown | null } | null } +/** + * V2-native records beyond noun/verb ops that a fact may carry through the + * ENCODER (types 6/7/8/9/10 of the v2 registry: embed markers, blob + * manifests, projection notes, bootstrap baselines). Encoder-ready by + * design; nothing produces them yet — the deferred-embed sidecar and blob + * lifecycle remodel onto these records in a later leg. + */ +export type FactMarkerRecord = + | EmbedPendingRecord + | EmbedLandedRecord + | BlobManifestRecord + | ProjectionNoteRecord + | BootstrapBaselineRecord + +/** + * Mints the dense integer handle for an entity/verb id at fact-append time — + * REQUIRED to be reproducible: a rebuilt id mapper must reproduce the same + * assignments exactly, so the only legal implementation delegates to the + * metadata index's id mapper (`getOrAssign`). Returns a POSITIVE bigint; a + * minter that cannot resolve its mapper throws — an int of 0 is never written. + */ +export type FactIntMinter = (kind: 'noun' | 'verb', id: string) => bigint + /** One committed generation, as scanned back out of the log. */ export interface CommitFact { generation: number @@ -90,6 +165,12 @@ export interface CommitFact { ops: FactOp[] meta?: Record blobHashes?: string[] + /** + * V2-native marker records riding this fact (see {@link FactMarkerRecord}). + * Optional and additive: absent on every v1 fact and on every fact the + * current writers produce; requires a v2 tail to encode. + */ + records?: FactMarkerRecord[] } /** The telemetry a scan batch carries (frozen shape). */ @@ -143,6 +224,12 @@ interface FactsManifest { /** The append target. Its true content is established by scanning (crash tolerance). */ tailSegment: string | null updatedAt: string + /** + * This brain's stable id (additive, v2 cutover): minted as a uuid at the + * first v2 tail creation and never changed; the `log.genesis` record + * carries it. Absent on logs that have never had a v2 tail. + */ + brainId?: string } /** The narrow byte-level storage surface the fact log rides. */ @@ -251,40 +338,339 @@ function decodeFact(payload: Uint8Array): CommitFact { } } +/** + * Deep-normalize a decoded v2 JSON position (metadata legs, meta maps, + * notes) back to plain-JSON values: the v2 codec decodes msgpack int64/uint64 + * as `bigint` (its u64 wire discipline), but canonical records are JSON — a + * metadata timestamp like `createdAt: 1786…` must come back as the NUMBER it + * was encoded from. Safe-range bigints narrow exactly; anything beyond the + * safe-integer range in a JSON position refuses loudly (it cannot have come + * from a JSON write). + */ +function normalizeWireJson(value: unknown): unknown { + if (typeof value === 'bigint') { + if (value > BigInt(Number.MAX_SAFE_INTEGER) || value < -BigInt(Number.MAX_SAFE_INTEGER)) { + throw new Error( + `fact log v2: decoded integer ${value} exceeds the JS safe-integer range in a JSON position` + ) + } + return Number(value) + } + if (Array.isArray(value)) return value.map(normalizeWireJson) + if (value && typeof value === 'object' && !(value instanceof Uint8Array)) { + const out: Record = {} + for (const [k, v] of Object.entries(value)) out[k] = normalizeWireJson(v) + return out + } + return value +} + +/** + * JSON-serialization equivalence for a v2 ENCODE-side JSON position: drop + * undefined-valued object keys and map undefined array elements to null — + * exactly what `JSON.stringify` does when canonical records are persisted. + * Commit facts are built from write-cache-WARM objects that may still carry + * undefined-valued engine keys (`service: undefined`, …) which the durable + * JSON never had; msgpack would preserve them as nil (the v1 capture's known + * wart), so the v2 capture — the future storage authority — sanitizes to the + * DURABLE truth instead. + */ +function toJsonSafe(value: unknown): unknown { + if (value === undefined) return null + if (Array.isArray(value)) return value.map((v) => (v === undefined ? null : toJsonSafe(v))) + if (value && typeof value === 'object' && !(value instanceof Uint8Array)) { + const out: Record = {} + for (const [k, v] of Object.entries(value)) { + if (v === undefined) continue + out[k] = toJsonSafe(v) + } + return out + } + return value +} + +/** Mirror of the storage layer's stored-timestamp normalization, minus its + * `Date.now()` fallback (a DECODER must be deterministic — an unreadable + * timestamp is omitted, and the divergence surfaces via the oracle). */ +function reconstructTimestamp(value: unknown): number | undefined { + if (typeof value === 'number' && value > 0) return value + if ( + value !== null && + typeof value === 'object' && + typeof (value as { seconds?: unknown }).seconds === 'number' + ) { + return (value as { seconds: number }).seconds * 1000 + } + return undefined +} + +/** + * Rebuild a noun's canonical VECTOR-FILE wrapper from a v2 after-image — + * the read-side of the hydration law. Canonical noun vector files hold the + * denormalized enumerable entity (`{id, vector, connections, level, type, + * …reserved fields…, metadata}` — the write path's composition); the v2 + * record deliberately carries only the ENTITY state (metadata leg + embedding + * floats), because connections/level are derived HNSW residue with their own + * rebuild paths (empty in every 8.x write) and the denormalized top-level + * fields are projections of the metadata leg. This reconstruction applies + * the SAME split/hydrate law the storage layer uses + * (`splitNounMetadataRecord` — the single source of truth in + * src/types/reservedFields.ts; field map mirrors + * `BaseStorage.hydrateNounWithMetadata`, undefined keys omitted exactly as + * JSON serialization omits them), so in the no-drift case the reconstructed + * wrapper digests byte-equal to canonical. A drifted denormalized copy + * surfaces as an oracle `state-differs` — named, never silently absorbed. + */ +function reconstructNounWrapper( + id: string, + metadataLeg: unknown, + floats: number[] +): Record { + const { reserved, custom } = splitNounMetadataRecord( + (metadataLeg ?? null) as Record | null + ) + const wrapper: Record = { + id, + vector: floats, + connections: {}, + level: 0, + type: (reserved.noun as string) || NounType.Thing + } + if (reserved.subtype !== undefined) wrapper.subtype = reserved.subtype + if (reserved.visibility !== undefined) wrapper.visibility = reserved.visibility + const createdAt = reconstructTimestamp(reserved.createdAt) + if (createdAt !== undefined) wrapper.createdAt = createdAt + const updatedAt = reconstructTimestamp(reserved.updatedAt) + if (updatedAt !== undefined) wrapper.updatedAt = updatedAt + if (reserved.confidence !== undefined) wrapper.confidence = reserved.confidence + if (reserved.weight !== undefined) wrapper.weight = reserved.weight + if (reserved.service !== undefined) wrapper.service = reserved.service + if (reserved.data !== undefined) wrapper.data = reserved.data + if (reserved.createdBy !== undefined) wrapper.createdBy = reserved.createdBy + wrapper._rev = typeof reserved._rev === 'number' ? reserved._rev : 1 + wrapper.metadata = custom + return wrapper +} + +/** Coerce a candidate embedding to `number[]`: plain arrays pass through + * (element-checked); numeric typed arrays (the JS HNSW rebuild path stores + * `Float32Array` vectors on the memory adapter) widen via `Array.from`. */ +function floatsOf(candidate: unknown, context: string): number[] | undefined { + if (Array.isArray(candidate)) { + for (const el of candidate) { + if (typeof el !== 'number') { + throw new Error(`fact log v2: ${context} vector carries a non-number element`) + } + } + return candidate as number[] + } + if (ArrayBuffer.isView(candidate) && !(candidate instanceof DataView)) { + return Array.from(candidate as unknown as ArrayLike) + } + return undefined +} + +/** Extract the embedding float array from a canonical vector value: a bare + * float array (or numeric typed array) passes through; a wrapper object + * yields its `vector` floats; `null` stays `null`; anything else refuses + * loudly. */ +function embeddingLegOf(value: unknown, context: string): number[] | null { + if (value === null || value === undefined) return null + const direct = floatsOf(value, context) + if (direct !== undefined) return direct + if (typeof value === 'object') { + const nested = floatsOf((value as { vector?: unknown }).vector, context) + if (nested !== undefined) return nested + } + throw new Error( + `fact log v2: ${context} has a canonical vector record with no float vector — ` + + `cannot encode its after-image` + ) +} + +/** + * Map one decoded v2 fact to the {@link CommitFact} shape every consumer + * already reads: noun/verb after-images and tombstones become ops (vector + * wrappers reconstructed — see {@link reconstructNounWrapper}); a + * `batch.meta` record becomes `meta` when the fact position carries none; + * `log.genesis` is log-level metadata (its width was verified at decode) and + * is not an op; marker records surface on the additive `records` field so + * nothing is silently dropped. Decoded JSON positions are normalized back + * from the codec's bigint discipline ({@link normalizeWireJson}). + */ +function commitFactFromV2(f: CommitFactV2): CommitFact { + const ops: FactOp[] = [] + const markers: FactMarkerRecord[] = [] + let batchMeta: Record | undefined + for (const r of f.records) { + switch (r.type) { + case 'noun.afterImage': { + const metadata = normalizeWireJson(r.metadata) ?? null + let vector: unknown | null = null + if (r.vectorLeg !== null) { + if (!Array.isArray(r.vectorLeg)) { + throw new Error( + `fact log v2: noun.afterImage ${r.id} carries a vector ref — this reader ` + + `resolves inline vectors only (refs are a later leg); refusing` + ) + } + vector = reconstructNounWrapper(r.id, metadata, r.vectorLeg) + } + ops.push({ kind: 'noun', id: r.id, record: { metadata, vector } }) + break + } + case 'noun.tombstone': + ops.push({ kind: 'noun', id: r.id, record: null }) + break + case 'verb.afterImage': { + const metadata = normalizeWireJson(r.metadata) ?? null + if (r.vectorLeg !== null && !Array.isArray(r.vectorLeg)) { + throw new Error( + `fact log v2: verb.afterImage ${r.id} carries a vector ref — this reader ` + + `resolves inline vectors only (refs are a later leg); refusing` + ) + } + // The canonical verb vector-file wrapper: endpoints + verb name ride + // as first-class v2 wire fields precisely so this reconstruction is + // exact ({id, vector, connections:{}, verb, sourceId, targetId} — + // verbs carry no `level`). + const vector: Record = { + id: r.id, + vector: r.vectorLeg ?? [], + connections: {}, + verb: r.verb, + sourceId: r.sourceId, + targetId: r.targetId + } + ops.push({ kind: 'verb', id: r.id, record: { metadata, vector } }) + break + } + case 'verb.tombstone': + ops.push({ kind: 'verb', id: r.id, record: null }) + break + case 'batch.meta': + batchMeta = normalizeWireJson(r.meta) as Record + break + case 'log.genesis': + break // the log's birth certificate — log-level metadata, not an op + case 'projection.note': + markers.push({ ...r, note: normalizeWireJson(r.note) as Record }) + break + case 'bootstrap.baseline': + markers.push({ ...r, metadata: normalizeWireJson(r.metadata) }) + break + default: + // embed.pending / embed.landed / blob.manifest carry no loose JSON maps. + markers.push(r) + break + } + } + const meta = f.meta ? (normalizeWireJson(f.meta) as Record) : batchMeta + return { + generation: f.generation, + timestamp: f.timestamp, + ops, + ...(meta ? { meta } : {}), + ...(f.blobHashes && f.blobHashes.length > 0 ? { blobHashes: f.blobHashes } : {}), + ...(markers.length > 0 ? { records: markers } : {}) + } +} + +/** One intact v2 frame's extent inside a segment (byte-slicing support). */ +interface V2FrameExtent { + /** Byte offset just past this frame. */ + end: number + /** The frame's generation (0 for pad filler). */ + generation: number + /** True when the frame is a pad (invisible filler). */ + isPad: boolean +} + +/** Walk a v2 segment's intact frames (torn-tail terminated), returning each + * frame's extent — the byte-level view truncation slices against, so kept + * frames are never re-encoded (byte-immutability of CRC-covered frames). */ +function walkV2Frames(bytes: Uint8Array): V2FrameExtent[] { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const extents: V2FrameExtent[] = [] + let offset = HEADER_BYTES + while (offset + FRAME_PREFIX_BYTES <= bytes.length) { + const length = view.getUint32(offset, true) + const expectedCrc = view.getUint32(offset + 4, true) + const start = offset + FRAME_PREFIX_BYTES + const end = start + length + if (end > bytes.length) break // torn tail + const payload = bytes.subarray(start, end) + if (crc32c(payload) !== expectedCrc) break // torn tail + const fact = decodeFormatFact(payload, FACT_LOG_FORMAT_V2, { + expectedIdSpaceWidth: 64 + }) as CommitFactV2 + extents.push({ end, generation: fact.generation, isPad: fact.records.length === 0 }) + offset = end + } + return extents +} + +/** + * The byte offset a v2 segment is cut at to keep exactly the facts with + * `generation ≤ keepThrough`: the end of the last kept FACT frame (pads + * between kept facts sit inside the retained span; pads after the cut are + * dropped and re-sealed at the next sync). When nothing is dropped the cut + * lands after the last intact frame — trailing pads retained, only a torn + * suffix (if any) removed. + */ +function v2CutOffset(extents: V2FrameExtent[], keepThrough: number): number { + let cut = HEADER_BYTES + let lastIntactEnd = HEADER_BYTES + for (const e of extents) { + lastIntactEnd = e.end + if (e.isPad) continue + if (e.generation <= keepThrough) { + cut = e.end + } else { + return cut // first beyond-keep fact: everything from here (pads included) goes + } + } + return lastIntactEnd +} + /** * Parse a segment's bytes: verify the header, then walk frames until the end * or a torn tail (length overrun / CRC mismatch), which terminates the walk — - * everything before it is intact. Returns the decoded facts plus the byte - * length of the VALID prefix (header + intact frames), which reconciliation - * uses to cut a torn tail without re-encoding. + * everything before it is intact. The header's formatVersion selects the + * decoder: the v1 walk below is byte-identical to the original v1 reader; + * v2 segments decode through the reference codec (`decodeGroupV2`, pads + * invisible, id-space width verified at 64 — a disagreeing genesis throws + * the codec's typed `GenesisWidthMismatchError`). Returns the decoded facts + * plus the byte length of the VALID prefix (header + intact frames), which + * reconciliation uses to cut a torn tail without re-encoding. */ function parseSegment( file: string, bytes: Uint8Array -): { facts: CommitFact[]; validBytes: number } { +): { facts: CommitFact[]; validBytes: number; formatVersion: number; sealSize?: number } { if (bytes.length < HEADER_BYTES) { prodLog.warn(`[FactLog] segment ${file} shorter than its header — treating as empty`) - return { facts: [], validBytes: 0 } + return { facts: [], validBytes: 0, formatVersion: 0 } } - for (let i = 0; i < MAGIC.length; i++) { - if (bytes[i] !== MAGIC[i]) { - throw new Error(`fact log: segment ${file} has a bad magic — not a fact segment`) - } + let header: { formatVersion: number; sealSize?: number } + try { + header = parseSegmentHeader(bytes.subarray(0, HEADER_BYTES)) + } catch (err) { + throw new Error(`fact log: segment ${file}: ${(err as Error).message}`) } - const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) - const version = view.getUint32(8, true) - if (version !== FACTS_FORMAT_VERSION) { - throw new Error( - `fact log: segment ${file} has formatVersion ${version}; this build reads ${FACTS_FORMAT_VERSION}` - ) - } - for (let i = 20; i < HEADER_BYTES; i++) { - if (bytes[i] !== 0) { - // Non-zero reserved bytes = a future format this build cannot verify. - throw new Error(`fact log: segment ${file} has non-zero reserved header bytes — unverifiable`) + + if (header.formatVersion === FACT_LOG_FORMAT_V2) { + const group = decodeGroupV2(bytes.subarray(HEADER_BYTES), { expectedIdSpaceWidth: 64 }) + return { + facts: group.facts.map(commitFactFromV2), + validBytes: HEADER_BYTES + group.validBytes, + formatVersion: FACT_LOG_FORMAT_V2, + sealSize: header.sealSize } } + // v1 walk — byte-identical to the original reader. + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength) const facts: CommitFact[] = [] let offset = HEADER_BYTES while (offset + FRAME_PREFIX_BYTES <= bytes.length) { @@ -298,7 +684,7 @@ function parseSegment( facts.push(decodeFact(payload)) offset = end } - return { facts, validBytes: offset } + return { facts, validBytes: offset, formatVersion: FACT_LOG_FORMAT_V1 } } /** @@ -318,18 +704,38 @@ export class FactLog { } /** Decoded facts of the TAIL segment (bounded by the rotation threshold). */ private tailFacts: CommitFact[] = [] - /** Byte size of the tail segment file (valid prefix). */ + /** Byte size of the tail segment file (valid prefix, pads included — + * pads count toward bytes but NEVER toward facts). */ private tailBytes = 0 /** Highest generation in the log (0 = empty). */ private head = 0 /** Segment paths appended since the last sync (the fsync batch). */ private readonly dirtySegments = new Set() + /** The TAIL segment's on-disk format version (selects the live encoder). */ + private tailVersion: number = FACT_LOG_FORMAT_V1 + /** The tail's sector-seal size (v2 tails; from its header on reopen). */ + private tailSealSize: number = DEFAULT_SEAL_SIZE + /** The v2 int minter (see {@link FactIntMinter}); null = v1 live writes. */ + private intMinter: FactIntMinter | null = null constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) { this.storage = storage this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES } + /** + * Install the v2 int minter — the capability gate for v2 LIVE WRITES. + * With a minter installed, every NEW tail segment writes the v2 format and + * after-image records carry minted dense ints; without one, live writes + * stay v1 (no authority could reproduce int assignments, and 0 is never + * written). The brain wires this from the metadata index's id mapper right + * after the index is ready; an existing v1 tail cuts over on the next + * append (empty tail: re-headed in place; non-empty: sealed by rotation). + */ + setIntMinter(mint: FactIntMinter): void { + this.intMinter = mint + } + /** The highest committed generation the log holds (0 = empty). */ headGeneration(): number { return this.head @@ -412,11 +818,17 @@ export class FactLog { const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` const bytes = await this.storage.readRawBytes(tailPath) if (bytes === null) { - // Manifest named a tail whose first byte never landed — an empty tail. + // Manifest named a tail whose first byte never landed — an empty + // tail. Its header (and format version) is established at the next + // append (see the tail-provisioning ladder there). this.tailFacts = [] this.tailBytes = 0 } else { - const { facts, validBytes } = parseSegment(this.manifest.tailSegment, bytes) + const parsed = parseSegment(this.manifest.tailSegment, bytes) + const { facts, validBytes } = parsed + this.tailVersion = + parsed.formatVersion === FACT_LOG_FORMAT_V2 ? FACT_LOG_FORMAT_V2 : FACT_LOG_FORMAT_V1 + this.tailSealSize = parsed.sealSize ?? DEFAULT_SEAL_SIZE const kept = facts.filter((f) => f.generation <= committedGeneration) if (kept.length !== facts.length || validBytes !== bytes.length) { const dropped = facts.length - kept.length @@ -426,7 +838,16 @@ export class FactLog { `${committedGeneration} from the tail (never committed)` ) } - await this.rewriteTail(kept) + if (this.tailVersion === FACT_LOG_FORMAT_V2) { + // V2: byte-slice at frame boundaries — CRC-covered frames are + // byte-immutable; a truncation never re-encodes what it keeps. + const cut = v2CutOffset(walkV2Frames(bytes), committedGeneration) + await this.storage.writeRawBytes(tailPath, bytes.subarray(0, cut)) + this.tailFacts = kept + this.tailBytes = cut + } else { + await this.rewriteTail(kept) + } } else { this.tailFacts = facts this.tailBytes = validBytes @@ -441,6 +862,15 @@ export class FactLog { * Append one committed generation's fact. NOT durable until {@link sync} — * the caller batches durability at its commit barrier (transact syncs in * the same call; Model-B group-commit syncs at flush). + * + * Tail provisioning (in order): a missing tail starts one; a named tail + * whose header never landed (manifest-first crash) gets its header now; an + * existing V1 tail cuts over to v2 once the minter is installed (empty: + * re-headed in place, non-empty: sealed by rotation — v1 segments are never + * rewritten); a full tail rotates. The frame then encodes in the TAIL's + * format: v2 tails carry after-image records with minted ints (and the + * genesis record on the very first fact of a brand-new log); v1 tails keep + * the v1 wire format byte-identically. */ async append(fact: CommitFact): Promise { if (fact.generation <= this.head) { @@ -450,10 +880,42 @@ export class FactLog { } if (this.manifest.tailSegment === null) { await this.startTail(fact.generation) + } else if (this.tailBytes === 0) { + await this.reinitializeTailHeader() + } else if (this.intMinter !== null && this.tailVersion === FACT_LOG_FORMAT_V1) { + if (this.tailFacts.length === 0 && this.tailBytes <= HEADER_BYTES) { + await this.upgradeEmptyTailToV2() + } else { + await this.rotate(fact.generation) + } } else if (this.tailBytes >= this.rotateBytes) { await this.rotate(fact.generation) } - const frame = encodeFrame(fact) + + let frame: Uint8Array + if (this.tailVersion === FACT_LOG_FORMAT_V2) { + const records = this.buildV2Records(fact) + if (this.needsGenesis()) { + if (this.ensureBrainId()) await this.persistManifest() + records.unshift(this.genesisRecord()) + } + frame = encodeFactV2({ + generation: fact.generation, + timestamp: fact.timestamp, + records, + ...(fact.meta ? { meta: toJsonSafe(fact.meta) as Record } : {}), + ...(fact.blobHashes && fact.blobHashes.length > 0 ? { blobHashes: fact.blobHashes } : {}) + }) + } else { + if (fact.records && fact.records.length > 0) { + throw new Error( + `fact log: marker records (${fact.records.map((r) => r.type).join(', ')}) require a ` + + `v2 tail segment — this log's tail is v1 (no int minter installed); refusing rather ` + + `than silently dropping them` + ) + } + frame = encodeFrame(fact) + } const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}` await this.storage.appendRawBytes(tailPath, frame) this.tailFacts.push(fact) @@ -462,8 +924,16 @@ export class FactLog { this.dirtySegments.add(tailPath) } - /** Fsync every segment appended since the last sync. */ + /** + * Fsync every segment appended since the last sync. SEALS AT SYNC: a v2 + * tail is first padded to its sector-seal boundary (one pad frame, + * invisible to readers; a gap smaller than the smallest constructible pad + * frame pads through one extra sector — the codec's rule), so every + * durability barrier leaves the tail sector-aligned: a torn page can only + * tear INSIDE the group being written, never a previously-sealed one. + */ async sync(): Promise { + await this.padTailToSealBoundary() if (this.dirtySegments.size === 0) return const paths = [...this.dirtySegments] this.dirtySegments.clear() @@ -671,7 +1141,25 @@ export class FactLog { `(head ${this.head}) — the fact to drop was already sealed; the log needs reopen` ) } - await this.rewriteTail(kept) + if (this.tailVersion === FACT_LOG_FORMAT_V2) { + // V2: byte-slice at frame boundaries (kept frames stay byte-identical; + // pads between kept facts are retained inside the prefix, trailing pads + // go and the next sync re-seals). The dropped frames may be unsynced — + // readRawBytes is read-after-write coherent over the append path. + const file = this.manifest.tailSegment + if (!file) return + const tailPath = `${FACTS_PREFIX}/${file}` + const bytes = await this.storage.readRawBytes(tailPath) + if (bytes === null) { + throw new Error(`fact log: dropAbove(${keepThrough}) cannot read the tail segment ${file}`) + } + const cut = v2CutOffset(walkV2Frames(bytes), keepThrough) + await this.storage.writeRawBytes(tailPath, bytes.subarray(0, cut)) + this.tailFacts = kept + this.tailBytes = cut + } else { + await this.rewriteTail(kept) + } this.head = this.computeHead() } @@ -684,25 +1172,42 @@ export class FactLog { return 0 } + /** The header bytes for a NEW tail: v2 whenever the minter is installed. */ + private newTailHeader(firstGeneration: number): Uint8Array { + return this.intMinter !== null + ? encodeSegmentHeaderV2(firstGeneration, DEFAULT_SEAL_SIZE) + : buildHeader(firstGeneration) + } + + /** Record the just-created tail's format in memory (mirrors its header). */ + private noteFreshTail(): void { + this.tailVersion = this.intMinter !== null ? FACT_LOG_FORMAT_V2 : FACT_LOG_FORMAT_V1 + this.tailSealSize = DEFAULT_SEAL_SIZE + } + /** Create the very first tail segment (manifest-first, then header bytes). */ private async startTail(firstGeneration: number): Promise { const file = segmentFileName(firstGeneration) this.manifest.tailSegment = file + if (this.intMinter !== null) this.ensureBrainId() await this.persistManifest() - await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, buildHeader(firstGeneration)) + await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, this.newTailHeader(firstGeneration)) this.tailFacts = [] this.tailBytes = HEADER_BYTES + this.noteFreshTail() } /** * Seal the tail into the manifest and start a new one. Manifest-first: the * flip both seals the old tail AND names the new one atomically, so no - * segment file ever exists unaccounted for. + * segment file ever exists unaccounted for. The NEW tail's format follows + * the minter gate ({@link newTailHeader}) — this is also the v1→v2 cutover + * seam for a non-empty v1 tail (sealed as-is, never rewritten). */ private async rotate(nextGeneration: number): Promise { const sealedFile = this.manifest.tailSegment if (!sealedFile) return - // Seal what the tail actually holds. + // Seal what the tail actually holds (sync() also sector-seals a v2 tail). await this.sync() // sealed segments are always fully durable const entry: SegmentEntry = { file: sealedFile, @@ -714,10 +1219,181 @@ export class FactLog { const newFile = segmentFileName(nextGeneration) this.manifest.segments.push(entry) this.manifest.tailSegment = newFile + if (this.intMinter !== null) this.ensureBrainId() await this.persistManifest() - await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, buildHeader(nextGeneration)) + await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, this.newTailHeader(nextGeneration)) this.tailFacts = [] this.tailBytes = HEADER_BYTES + this.noteFreshTail() + } + + /** + * The v1→v2 cutover for an EMPTY v1 tail: re-head it in place (nothing but + * the 32-byte header exists, so no v1 frame is ever rewritten). Also the + * cheapest cutover shape: brand-new brains whose first tail predates the + * minter installation converge here on their first post-install append. + */ + private async upgradeEmptyTailToV2(): Promise { + const file = this.manifest.tailSegment + if (!file) return + if (this.ensureBrainId()) await this.persistManifest() + const first = this.segmentFirstGenerationFromName(file) + const path = `${FACTS_PREFIX}/${file}` + await this.storage.writeRawBytes(path, encodeSegmentHeaderV2(first, DEFAULT_SEAL_SIZE)) + this.tailBytes = HEADER_BYTES + this.tailVersion = FACT_LOG_FORMAT_V2 + this.tailSealSize = DEFAULT_SEAL_SIZE + this.dirtySegments.add(path) + } + + /** + * A manifest-named tail whose header never landed (crash between the + * manifest flip and the first header byte — previously this appended + * frames into a headerless file the next open could not parse): write the + * header now, in the CURRENT format gate. + */ + private async reinitializeTailHeader(): Promise { + const file = this.manifest.tailSegment + if (!file) return + if (this.intMinter !== null && this.ensureBrainId()) await this.persistManifest() + const first = this.segmentFirstGenerationFromName(file) + const path = `${FACTS_PREFIX}/${file}` + await this.storage.writeRawBytes(path, this.newTailHeader(first)) + this.tailBytes = HEADER_BYTES + this.noteFreshTail() + this.dirtySegments.add(path) + } + + /** True when the NEXT appended fact is the first fact of a brand-new v2 + * log — the one that must open with the log.genesis record. */ + private needsGenesis(): boolean { + return ( + this.tailVersion === FACT_LOG_FORMAT_V2 && + this.manifest.segments.length === 0 && + this.tailFacts.length === 0 + ) + } + + /** Mint the brain id into the manifest if absent; true when it changed. */ + private ensureBrainId(): boolean { + if (this.manifest.brainId) return false + this.manifest.brainId = uuidv4() + return true + } + + /** The log's birth certificate (id-space width 64 — the only width this + * writer mints; a reader expecting another width refuses at decode). */ + private genesisRecord(): LogRecord { + const brainId = this.manifest.brainId + if (!brainId) { + throw new Error( + 'fact log v2: genesis requires a brainId in the facts manifest — invariant violated' + ) + } + return { type: 'log.genesis', idSpaceWidth: 64, brainId, createdAt: Date.now() } + } + + /** + * Convert one CommitFact's ops (+ optional marker records) to v2 wire + * records, MINTING ints at append time: entity/verb ints come from the + * injected minter (the metadata index's id mapper — the one authority a + * rebuild reproduces exactly). Verb endpoints and the verb name ride as + * first-class wire fields, lifted from the canonical verb vector wrapper. + * Every refusal here is loud — an after-image without a mintable int, a + * verb without endpoints, or a vector record without floats fails the + * WRITE, never writes a 0. + */ + private buildV2Records(fact: CommitFact): LogRecord[] { + const mint = (kind: 'noun' | 'verb', id: string): bigint => { + if (this.intMinter === null) { + throw new Error( + `fact log v2: no int minter is installed — cannot mint the ${kind} int for ${id}; ` + + `refusing to write a v2 after-image (an int of 0 is never written)` + ) + } + const minted = this.intMinter(kind, id) + if (typeof minted !== 'bigint' || minted <= 0n) { + throw new Error( + `fact log v2: the int minter returned ${String(minted)} for ${kind} ${id} — ` + + `minted ints are positive bigints; refusing to write` + ) + } + return minted + } + + const records: LogRecord[] = [] + for (const op of fact.ops) { + if (op.kind === 'noun') { + if (op.record === null) { + records.push({ type: 'noun.tombstone', id: op.id }) + continue + } + records.push({ + type: 'noun.afterImage', + id: op.id, + entityInt: mint('noun', op.id), + metadata: toJsonSafe(op.record.metadata ?? null), + vectorLeg: embeddingLegOf(op.record.vector, `noun ${op.id}`) + }) + } else { + if (op.record === null) { + records.push({ type: 'verb.tombstone', id: op.id }) + continue + } + const wrapper = op.record.vector as Record | null + const verbName = wrapper?.verb + const sourceId = wrapper?.sourceId + const targetId = wrapper?.targetId + if ( + typeof verbName !== 'string' || + typeof sourceId !== 'string' || + typeof targetId !== 'string' + ) { + throw new Error( + `fact log v2: verb ${op.id} has no canonical endpoints (verb/sourceId/targetId ` + + `live in its vector record, which is missing or torn) — refusing to write an ` + + `after-image that could not be replayed` + ) + } + const floats = floatsOf(wrapper?.vector, `verb ${op.id}`) ?? [] + records.push({ + type: 'verb.afterImage', + id: op.id, + verbInt: mint('verb', op.id), + metadata: toJsonSafe(op.record.metadata ?? null), + vectorLeg: floats, + verb: verbName, + sourceId, + sourceInt: mint('noun', sourceId), + targetId, + targetInt: mint('noun', targetId) + }) + } + } + for (const marker of fact.records ?? []) records.push(marker) + return records + } + + /** + * Pad a v2 tail to its next sector-seal boundary with ONE pad frame — + * called from {@link sync} so alignment holds at every durability barrier. + * Pads count toward {@link tailBytes} but never toward facts (they are + * invisible to every reader); a gap smaller than the smallest constructible + * pad frame pads through one extra sector (the codec's rule). No-op for v1 + * tails, empty tails, and already-aligned tails. + */ + private async padTailToSealBoundary(): Promise { + if (this.tailVersion !== FACT_LOG_FORMAT_V2) return + const file = this.manifest.tailSegment + if (!file || this.tailBytes <= HEADER_BYTES) return + const remainder = this.tailBytes % this.tailSealSize + if (remainder === 0) return + let padBytes = this.tailSealSize - remainder + if (padBytes < minPadFrameBytes()) padBytes += this.tailSealSize + const tailPath = `${FACTS_PREFIX}/${file}` + await this.storage.appendRawBytes(tailPath, encodePadFrame(padBytes)) + this.tailBytes += padBytes + this.dirtySegments.add(tailPath) } /** Atomically persist the manifest (write-new → fsync → rename downstream). */ @@ -746,17 +1422,24 @@ export class FactLog { this.tailBytes = total } - /** Cut a SEALED segment back to `committedGeneration` (atomic replace). */ + /** Cut a SEALED segment back to `committedGeneration` (atomic replace). + * v2 segments byte-slice at frame boundaries (kept frames — pads + * included — are never re-encoded); the v1 re-encode path is unchanged. */ private async truncateSegmentTo(file: string, committedGeneration: number): Promise { const path = `${FACTS_PREFIX}/${file}` const bytes = await this.storage.readRawBytes(path) if (bytes === null) return - const { facts } = parseSegment(file, bytes) + const { facts, formatVersion } = parseSegment(file, bytes) const kept = facts.filter((f) => f.generation <= committedGeneration) prodLog.warn( `[FactLog] truncating sealed segment ${file} to generation ${committedGeneration} ` + `(${facts.length - kept.length} uncommitted fact(s) dropped)` ) + if (formatVersion === FACT_LOG_FORMAT_V2) { + const cut = v2CutOffset(walkV2Frames(bytes), committedGeneration) + await this.storage.writeRawBytes(path, bytes.subarray(0, cut)) + return + } const first = kept[0]?.generation ?? this.segmentFirstGenerationFromName(file) const parts: Uint8Array[] = [buildHeader(first)] for (const f of kept) parts.push(encodeFrame(f)) diff --git a/src/db/factLogFormat.ts b/src/db/factLogFormat.ts index 0ca86410..8642d890 100644 --- a/src/db/factLogFormat.ts +++ b/src/db/factLogFormat.ts @@ -26,9 +26,21 @@ * position 2 is `records`, not v1's `ops`) * * fact := [ generation:u64, timestamp:u64, records, meta|nil, blobHashes|nil ] - * record := [ recordType:u8, recordVersion:u8, ...type-specific fields ] + * record := [ recordType:u8, recordVersion:u8, cipherFlag:u8, keyId:bin16|nil, + * ...type-specific fields ] * - * Record type registry (all recordVersion = 1): + * `cipherFlag`/`keyId` are RESERVED crypto envelope fields: `0`/`nil` (a + * plaintext record) is the ONLY legal combination this release writes or + * reads. Any nonzero cipherFlag or non-nil keyId refuses with the typed + * {@link UnknownLogRecordError} ("encrypted records need a newer reader") — + * so record-level encryption can land later without a format-version bump on + * the one compat surface. No crypto logic exists here; the bytes are reserved + * only. Pad records (type 0) are exempt: they are skipped WHOLESALE as + * length-only filler, so their fields beyond [type, version] are never + * inspected (this keeps pad frames byte-stable across the envelope change). + * + * Record type registry (all recordVersion = 1; type-specific fields listed — + * every record carries the 4-field envelope above first): * * 0 pad [] — length-only filler; readers SKIP; crc-covered * 1 noun.afterImage [id bin16, entityInt u64, metadata, vectorLeg] @@ -109,6 +121,13 @@ export const DEFAULT_SEAL_SIZE = 4096 /** The record version this reader knows (all registry types are version 1). */ export const LOG_RECORD_VERSION = 1 +/** + * The only legal `cipherFlag` value this release: plaintext. The encoder + * always writes it (with a nil keyId); the decoder refuses anything else + * with {@link UnknownLogRecordError} — encrypted records need a newer reader. + */ +export const LOG_RECORD_CIPHER_PLAINTEXT = 0 + /** The v2 record-type registry — wire codes for every record type. */ export const LOG_RECORD_TYPES = { PAD: 0, @@ -648,22 +667,31 @@ function decodeVectorLeg(wire: unknown, context: string): VectorLeg { // Record encode/decode // --------------------------------------------------------------------------- -/** Encode one record into its positional wire array. */ +/** + * Encode one record into its positional wire array. Every record leads with + * the 4-field envelope [type, version, cipherFlag, keyId]; this release + * writes cipherFlag {@link LOG_RECORD_CIPHER_PLAINTEXT} and a nil keyId + * always (the fields are crypto-RESERVED, carrying no logic yet). + */ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefined): unknown[] { const T = LOG_RECORD_TYPES const V = LOG_RECORD_VERSION + const C = LOG_RECORD_CIPHER_PLAINTEXT + const K = null // keyId: nil until record-level encryption exists switch (record.type) { case 'noun.afterImage': return [ T.NOUN_AFTER_IMAGE, V, + C, + K, uuidToBytes(record.id), toWireU64(record.entityInt, 'entityInt'), record.metadata ?? null, encodeVectorLeg(record.vectorLeg, options, `noun.afterImage ${record.id}`) ] case 'noun.tombstone': - return [T.NOUN_TOMBSTONE, V, uuidToBytes(record.id)] + return [T.NOUN_TOMBSTONE, V, C, K, uuidToBytes(record.id)] case 'verb.afterImage': { if (typeof record.verb !== 'string' || record.verb.length === 0) { throw new Error(`fact log v2: verb.afterImage ${record.id} needs a non-empty verb name`) @@ -671,6 +699,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine return [ T.VERB_AFTER_IMAGE, V, + C, + K, uuidToBytes(record.id), toWireU64(record.verbInt, 'verbInt'), record.metadata ?? null, @@ -683,16 +713,18 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine ] } case 'verb.tombstone': - return [T.VERB_TOMBSTONE, V, uuidToBytes(record.id)] + return [T.VERB_TOMBSTONE, V, C, K, uuidToBytes(record.id)] case 'batch.meta': if (!isPlainMap(record.meta)) { throw new Error('fact log v2: batch.meta requires a map') } - return [T.BATCH_META, V, record.meta] + return [T.BATCH_META, V, C, K, record.meta] case 'embed.pending': return [ T.EMBED_PENDING, V, + C, + K, uuidToBytes(record.id), toWireU64(record.enqueuedAt, 'enqueuedAt') ] @@ -703,7 +735,7 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine `refs and nil are not allowed here` ) } - return [T.EMBED_LANDED, V, uuidToBytes(record.id), record.vector] + return [T.EMBED_LANDED, V, C, K, uuidToBytes(record.id), record.vector] } case 'blob.manifest': { if (typeof record.mimeType !== 'string') { @@ -715,6 +747,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine return [ T.BLOB_MANIFEST, V, + C, + K, hashToBytes(record.hash), toWireU64(record.size, 'blob size'), record.mimeType, @@ -725,7 +759,7 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine if (!isPlainMap(record.note)) { throw new Error('fact log v2: projection.note requires a map') } - return [T.PROJECTION_NOTE, V, record.note] + return [T.PROJECTION_NOTE, V, C, K, record.note] case 'bootstrap.baseline': { if (record.kind !== 'noun' && record.kind !== 'verb') { throw new Error(`fact log v2: bootstrap.baseline kind must be 'noun' or 'verb'`) @@ -733,6 +767,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine return [ T.BOOTSTRAP_BASELINE, V, + C, + K, uuidToBytes(record.id), record.kind === 'noun' ? 0 : 1, record.metadata ?? null, @@ -748,6 +784,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine return [ T.LOG_GENESIS, V, + C, + K, record.idSpaceWidth, uuidToBytes(record.brainId), toWireU64(record.createdAt, 'createdAt') @@ -763,35 +801,39 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine } } -/** Exact wire arity per record type (envelope of 2 + type-specific fields). */ +/** Exact wire arity per record type (envelope of 4 + type-specific fields). */ const RECORD_ARITY: Record = { - [LOG_RECORD_TYPES.NOUN_AFTER_IMAGE]: 6, - [LOG_RECORD_TYPES.NOUN_TOMBSTONE]: 3, - [LOG_RECORD_TYPES.VERB_AFTER_IMAGE]: 11, - [LOG_RECORD_TYPES.VERB_TOMBSTONE]: 3, - [LOG_RECORD_TYPES.BATCH_META]: 3, - [LOG_RECORD_TYPES.EMBED_PENDING]: 4, - [LOG_RECORD_TYPES.EMBED_LANDED]: 4, - [LOG_RECORD_TYPES.BLOB_MANIFEST]: 6, - [LOG_RECORD_TYPES.PROJECTION_NOTE]: 3, - [LOG_RECORD_TYPES.BOOTSTRAP_BASELINE]: 6, - [LOG_RECORD_TYPES.LOG_GENESIS]: 5 + [LOG_RECORD_TYPES.NOUN_AFTER_IMAGE]: 8, + [LOG_RECORD_TYPES.NOUN_TOMBSTONE]: 5, + [LOG_RECORD_TYPES.VERB_AFTER_IMAGE]: 13, + [LOG_RECORD_TYPES.VERB_TOMBSTONE]: 5, + [LOG_RECORD_TYPES.BATCH_META]: 5, + [LOG_RECORD_TYPES.EMBED_PENDING]: 6, + [LOG_RECORD_TYPES.EMBED_LANDED]: 6, + [LOG_RECORD_TYPES.BLOB_MANIFEST]: 8, + [LOG_RECORD_TYPES.PROJECTION_NOTE]: 5, + [LOG_RECORD_TYPES.BOOTSTRAP_BASELINE]: 8, + [LOG_RECORD_TYPES.LOG_GENESIS]: 7 } /** * Decode one wire record. Returns `null` for pads (skipped by definition). * Unknown type / newer version throw {@link UnknownLogRecordError} — never - * skip-and-continue. + * skip-and-continue. The reserved crypto envelope is verified BEFORE the + * arity check (an encrypted record's field layout is a newer reader's + * business, not a malformed-record error): any nonzero cipherFlag or non-nil + * keyId refuses with the same typed error class. */ function decodeRecord(raw: unknown): LogRecord | null { if (!Array.isArray(raw) || raw.length < 2) { - throw new Error('fact log v2: malformed record envelope (need [type, version, ...])') + throw new Error('fact log v2: malformed record envelope (need [type, version, cipherFlag, keyId, ...])') } const recordType = wireToU8(raw[0], 'recordType') const recordVersion = wireToU8(raw[1], 'recordVersion') if (recordType === LOG_RECORD_TYPES.PAD) { - // Length-only filler: skipped wholesale, filler fields never inspected. + // Length-only filler: skipped wholesale, filler fields never inspected + // (pads therefore carry no crypto envelope — by definition, not omission). return null } const arity = RECORD_ARITY[recordType] @@ -814,6 +856,20 @@ function decodeRecord(raw: unknown): LogRecord | null { if (recordVersion !== LOG_RECORD_VERSION) { throw new Error(`fact log v2: record type ${recordType} has invalid record version ${recordVersion}`) } + if (raw.length < 4) { + throw new Error('fact log v2: malformed record envelope (need [type, version, cipherFlag, keyId, ...])') + } + const cipherFlag = wireToU8(raw[2], 'cipherFlag') + const keyId = raw[3] + if (cipherFlag !== LOG_RECORD_CIPHER_PLAINTEXT || (keyId !== null && keyId !== undefined)) { + throw new UnknownLogRecordError( + recordType, + recordVersion, + `fact log v2: record type ${recordType} carries cipherFlag ${cipherFlag}` + + `${keyId !== null && keyId !== undefined ? ' and a keyId' : ''} — ` + + `encrypted records need a newer reader` + ) + } if (raw.length !== arity) { throw new Error( `fact log v2: record type ${recordType} expects ${arity} wire fields; got ${raw.length}` @@ -824,94 +880,94 @@ function decodeRecord(raw: unknown): LogRecord | null { case LOG_RECORD_TYPES.NOUN_AFTER_IMAGE: return { type: 'noun.afterImage', - id: bytesToUuid(raw[2], 'noun.afterImage id'), - entityInt: wireToBigint(raw[3], 'entityInt'), - metadata: raw[4] ?? null, - vectorLeg: decodeVectorLeg(raw[5], 'noun.afterImage') + id: bytesToUuid(raw[4], 'noun.afterImage id'), + entityInt: wireToBigint(raw[5], 'entityInt'), + metadata: raw[6] ?? null, + vectorLeg: decodeVectorLeg(raw[7], 'noun.afterImage') } case LOG_RECORD_TYPES.NOUN_TOMBSTONE: - return { type: 'noun.tombstone', id: bytesToUuid(raw[2], 'noun.tombstone id') } + return { type: 'noun.tombstone', id: bytesToUuid(raw[4], 'noun.tombstone id') } case LOG_RECORD_TYPES.VERB_AFTER_IMAGE: { - if (typeof raw[6] !== 'string') { + if (typeof raw[8] !== 'string') { throw new Error('fact log v2: verb.afterImage verb name is not a string') } return { type: 'verb.afterImage', - id: bytesToUuid(raw[2], 'verb.afterImage id'), - verbInt: wireToBigint(raw[3], 'verbInt'), - metadata: raw[4] ?? null, - vectorLeg: decodeVectorLeg(raw[5], 'verb.afterImage'), - verb: raw[6], - sourceId: bytesToUuid(raw[7], 'verb.afterImage sourceId'), - sourceInt: wireToBigint(raw[8], 'sourceInt'), - targetId: bytesToUuid(raw[9], 'verb.afterImage targetId'), - targetInt: wireToBigint(raw[10], 'targetInt') + id: bytesToUuid(raw[4], 'verb.afterImage id'), + verbInt: wireToBigint(raw[5], 'verbInt'), + metadata: raw[6] ?? null, + vectorLeg: decodeVectorLeg(raw[7], 'verb.afterImage'), + verb: raw[8], + sourceId: bytesToUuid(raw[9], 'verb.afterImage sourceId'), + sourceInt: wireToBigint(raw[10], 'sourceInt'), + targetId: bytesToUuid(raw[11], 'verb.afterImage targetId'), + targetInt: wireToBigint(raw[12], 'targetInt') } } case LOG_RECORD_TYPES.VERB_TOMBSTONE: - return { type: 'verb.tombstone', id: bytesToUuid(raw[2], 'verb.tombstone id') } + return { type: 'verb.tombstone', id: bytesToUuid(raw[4], 'verb.tombstone id') } case LOG_RECORD_TYPES.BATCH_META: { - if (!isPlainMap(raw[2])) throw new Error('fact log v2: batch.meta payload is not a map') - return { type: 'batch.meta', meta: raw[2] } + if (!isPlainMap(raw[4])) throw new Error('fact log v2: batch.meta payload is not a map') + return { type: 'batch.meta', meta: raw[4] } } case LOG_RECORD_TYPES.EMBED_PENDING: return { type: 'embed.pending', - id: bytesToUuid(raw[2], 'embed.pending id'), - enqueuedAt: wireToNumber(raw[3], 'enqueuedAt') + id: bytesToUuid(raw[4], 'embed.pending id'), + enqueuedAt: wireToNumber(raw[5], 'enqueuedAt') } case LOG_RECORD_TYPES.EMBED_LANDED: { - const leg = decodeVectorLeg(raw[3], 'embed.landed') + const leg = decodeVectorLeg(raw[5], 'embed.landed') if (!Array.isArray(leg)) { throw new Error( 'fact log v2: embed.landed must carry an INLINE float vector — refs and nil are not allowed here' ) } - return { type: 'embed.landed', id: bytesToUuid(raw[2], 'embed.landed id'), vector: leg } + return { type: 'embed.landed', id: bytesToUuid(raw[4], 'embed.landed id'), vector: leg } } case LOG_RECORD_TYPES.BLOB_MANIFEST: { - if (typeof raw[4] !== 'string') { + if (typeof raw[6] !== 'string') { throw new Error('fact log v2: blob.manifest mimeType is not a string') } - const refOp = wireToU8(raw[5], 'refOp') + const refOp = wireToU8(raw[7], 'refOp') if (refOp !== 0 && refOp !== 1) { throw new Error(`fact log v2: blob.manifest refOp must be 0 (add) or 1 (release); got ${refOp}`) } return { type: 'blob.manifest', - hash: bytesToHash(raw[2]), - size: wireToNumber(raw[3], 'blob size'), - mimeType: raw[4], + hash: bytesToHash(raw[4]), + size: wireToNumber(raw[5], 'blob size'), + mimeType: raw[6], refOp: refOp === 0 ? 'add' : 'release' } } case LOG_RECORD_TYPES.PROJECTION_NOTE: { - if (!isPlainMap(raw[2])) throw new Error('fact log v2: projection.note payload is not a map') - return { type: 'projection.note', note: raw[2] } + if (!isPlainMap(raw[4])) throw new Error('fact log v2: projection.note payload is not a map') + return { type: 'projection.note', note: raw[4] } } case LOG_RECORD_TYPES.BOOTSTRAP_BASELINE: { - const kind = wireToU8(raw[3], 'bootstrap.baseline kind') + const kind = wireToU8(raw[5], 'bootstrap.baseline kind') if (kind !== 0 && kind !== 1) { throw new Error(`fact log v2: bootstrap.baseline kind must be 0 (noun) or 1 (verb); got ${kind}`) } return { type: 'bootstrap.baseline', - id: bytesToUuid(raw[2], 'bootstrap.baseline id'), + id: bytesToUuid(raw[4], 'bootstrap.baseline id'), kind: kind === 0 ? 'noun' : 'verb', - metadata: raw[4] ?? null, - vectorLeg: decodeVectorLeg(raw[5], 'bootstrap.baseline') + metadata: raw[6] ?? null, + vectorLeg: decodeVectorLeg(raw[7], 'bootstrap.baseline') } } case LOG_RECORD_TYPES.LOG_GENESIS: { - const width = wireToU8(raw[2], 'idSpaceWidth') + const width = wireToU8(raw[4], 'idSpaceWidth') if (width !== 32 && width !== 64) { throw new Error(`fact log v2: log.genesis idSpaceWidth must be 32 or 64; got ${width}`) } return { type: 'log.genesis', idSpaceWidth: width, - brainId: bytesToUuid(raw[3], 'log.genesis brainId'), - createdAt: wireToNumber(raw[4], 'createdAt') + brainId: bytesToUuid(raw[5], 'log.genesis brainId'), + createdAt: wireToNumber(raw[6], 'createdAt') } } default: @@ -944,8 +1000,12 @@ export function encodeFactV2(fact: CommitFactV2, options?: EncodeFactV2Options): if (!Number.isSafeInteger(fact.timestamp) || fact.timestamp < 0) { throw new Error(`fact log v2: timestamp must be a non-negative integer; got ${fact.timestamp}`) } - if (!Array.isArray(fact.records) || fact.records.length === 0) { - throw new Error('fact log v2: a fact must carry at least one record') + // records MAY be empty: a committed generation whose ops all collapsed + // (e.g. a batch whose relates deduped to no-ops) is still a real + // generation — v1 encoded empty ops the same way; refusing here would + // fork the two formats' commit semantics. + if (!Array.isArray(fact.records)) { + throw new Error('fact log v2: records must be an array') } if (fact.meta !== undefined && !isPlainMap(fact.meta)) { throw new Error('fact log v2: fact meta must be a map when present') @@ -1092,9 +1152,15 @@ function decodeFactV2(payload: Uint8Array, options?: DecodeFactV2Options): Commi // Sector seals // --------------------------------------------------------------------------- -/** Smallest constructible pad frame (envelope + bare pad record), memoized. */ +/** + * Smallest constructible pad frame in bytes (frame prefix + the bare pad + * record fact), memoized. Exported for streaming writers that pad an + * append-only tail to a seal boundary: a gap smaller than this cannot hold + * any frame, so the writer pads through one extra sector (the same rule + * {@link sealGroup} applies). + */ let minPadFrameBytesMemo: number | null = null -function minPadFrameBytes(): number { +export function minPadFrameBytes(): number { if (minPadFrameBytesMemo === null) { minPadFrameBytesMemo = FRAME_PREFIX_BYTES + @@ -1145,6 +1211,25 @@ function buildPadFrame(totalBytes: number): Uint8Array { return buildFrame(payload) } +/** + * Build a pad frame of EXACTLY `totalBytes` — the streaming-append counterpart + * of {@link sealGroup} for writers that append pads directly to a live tail + * instead of sealing an in-memory group. Refuses sizes smaller than the + * smallest constructible pad frame ({@link minPadFrameBytes}); readers skip + * the result by definition (a type-0 record is length-only filler). + * + * @param totalBytes - The exact frame size to construct (prefix included). + * @returns The complete pad frame bytes. + */ +export function encodePadFrame(totalBytes: number): Uint8Array { + if (!Number.isInteger(totalBytes) || totalBytes < minPadFrameBytes()) { + throw new Error( + `fact log v2: a pad frame must be at least ${minPadFrameBytes()} bytes; got ${totalBytes}` + ) + } + return buildPadFrame(totalBytes) +} + /** * Seal a group of frames to a sector boundary: concatenate the frames and pad * to the next `sealSize` multiple with ONE pad frame. An already-aligned diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 663784c6..2f623e3b 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -46,7 +46,13 @@ import type { TxLogEntry } from './types.js' import { readLogAuthority } from './logAuthority.js' -import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js' +import { + FactLog, + storageSupportsFactLog, + type CommitFact, + type FactOp, + type FactIntMinter +} from './factLog.js' import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js' import { crc32c } from '../utils/crc32c.js' @@ -182,6 +188,22 @@ export class GenerationStore { this.logDurability = mode } + /** + * The fact log's v2 int minter — injected by the OWNER (brainy wires the + * metadata index's id mapper here right after the index is ready), because + * this store cannot know the mapper. With the minter installed, new fact + * segments write the v2 format and after-image records carry minted dense + * ints reproducible by an id-mapper rebuild. Survives reopen: `open()` + * re-installs it on the fresh {@link FactLog} instance. + */ + private intMinter: FactIntMinter | null = null + + /** Install the fact log's v2 int minter (see {@link intMinter}). */ + setIntMinter(mint: FactIntMinter): void { + this.intMinter = mint + this.factLog?.setIntMinter(mint) + } + /** Latest reserved/observed generation (≥ {@link committed}). */ private counter = 0 /** Committed-transaction watermark (manifest generation). */ @@ -493,6 +515,7 @@ export class GenerationStore { // hosts no fact log (readers fall back to canonical enumeration). if (storageSupportsFactLog(this.storage)) { this.factLog = new FactLog(this.storage) + if (this.intMinter) this.factLog.setIntMinter(this.intMinter) // LOG-AUTHORITY REPLAY (durable-at-ack's recovery half): when this // brain's stored authority is the log, an intact fact ABOVE the // manifest is an ACKED write whose canonical bytes may not have diff --git a/tests/integration/fact-log-v2-cutover.test.ts b/tests/integration/fact-log-v2-cutover.test.ts new file mode 100644 index 00000000..6c05ef42 --- /dev/null +++ b/tests/integration/fact-log-v2-cutover.test.ts @@ -0,0 +1,389 @@ +/** + * @module tests/integration/fact-log-v2-cutover + * @description The fact log's LIVE WRITE FORMAT cutover to v2, end-to-end + * through real brains: (a) a NEW brain's tail segment carries a v2 header + * (formatVersion 2, sealSize 4096), opens with the log.genesis record + * (id-space width 64 + the manifest-persisted brainId), and scanFacts yields + * the same CommitFact shape a v1 brain would — reconstruction included, + * proven by digest-equality against canonical after a reopen; (b) MIXED + * logs: an existing v1 segment stays readable forever beside a v2 tail + * (cutover-by-rotation; the v1 segment is never rewritten); (c) MINT: + * after-image records carry the metadata index id mapper's exact int + * assignments (white-box compare); (d) SEALS: every flush leaves the tail + * sector-aligned, and pads are invisible to scans; (e) REPLAY: the + * log-authority recovery path resurrects an acked write from a v2 tail + * after a crash-style abandon. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as path from 'node:path' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { + parseSegmentHeader, + decodeGroupV2, + SEGMENT_HEADER_BYTES, + FACT_LOG_FORMAT_V1, + FACT_LOG_FORMAT_V2, + type LogGenesisRecord, + type NounAfterImageRecord +} from '../../src/db/factLogFormat.js' +import type { CommitFact, FactIntMinter, FactLog } from '../../src/db/factLog.js' +import { + makeTempDir, + openBrain, + storeOf, + abandonAsCrashed, + factGenerations, + vec, + uid +} from '../helpers/durabilityKillMatrix.js' + +/** The VFS root — created at init by a baseline (generation-less) write. */ +const VFS_ROOT = '00000000-0000-0000-0000-000000000000' +const FACTS_DIR = ['_generations', 'facts'] as const +const MANIFEST_PATH = '_generations/facts/manifest.json' + +/** White-box internals this suite instruments. */ +type BrainInternals = { + storage: { + readRawObject(p: string): Promise + readNounRaw(id: string): Promise<{ metadata: unknown | null; vector: unknown | null }> + } + metadataIndex: { + getIdMapper(): { getInt(uuid: string): number | undefined } + } +} +const internals = (brain: Brainy): BrainInternals => brain as unknown as BrainInternals + +/** The facts manifest as stored (additive brainId included). */ +interface StoredFactsManifest { + segments: Array<{ file: string }> + tailSegment: string | null + brainId?: string +} + +async function readManifest(brain: Brainy): Promise { + const manifest = (await internals(brain).storage.readRawObject( + MANIFEST_PATH + )) as StoredFactsManifest | null + expect(manifest, 'the facts manifest exists').toBeTruthy() + return manifest! +} + +/** Raw on-disk bytes of one fact segment file. */ +function segmentBytes(dir: string, file: string): Uint8Array { + return new Uint8Array(fs.readFileSync(path.join(dir, ...FACTS_DIR, file))) +} + +async function allFacts(brain: Brainy): Promise { + const scan = (brain as unknown as { scanFacts(): { batches(): AsyncGenerator<{ facts: CommitFact[] }> } | null }).scanFacts() + expect(scan, 'this storage hosts a fact log').not.toBeNull() + const facts: CommitFact[] = [] + for await (const batch of scan!.batches()) facts.push(...batch.facts) + return facts +} + +/** The live FactLog instance (white-box: the minter strip in scenario b). */ +function factLogOf(brain: Brainy): FactLog & { intMinter: FactIntMinter | null } { + const log = storeOf(brain).getFactLog() + expect(log, 'filesystem storage hosts a fact log').not.toBeNull() + return log as FactLog & { intMinter: FactIntMinter | null } +} + +describe('fact log v2 cutover — live writes land in the v2 segment format', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + const trackDir = (): string => { + const dir = makeTempDir() + dirs.push(dir) + return dir + } + const track = (brain: Brainy): Brainy => { + brains.push(brain) + return brain + } + + afterEach(async () => { + for (const b of brains.splice(0)) { + await (b as unknown as { close?: () => Promise }).close?.().catch(() => {}) + } + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) + }) + + it('(a) NEW BRAIN: v2 tail header, genesis-first, and scanFacts parity with canonical across a reopen', async () => { + const dir = trackDir() + const brain = track(await openBrain(dir)) + const idA = uid('v2-new-a') + const idB = uid('v2-new-b') + await brain.add({ id: idA, data: 'alpha', type: NounType.Document, vector: vec(1), metadata: { n: 1 } }) + await brain.add({ id: idB, data: 'beta', type: NounType.Document, vector: vec(2), metadata: { n: 2 } }) + await brain.flush() + + // The tail segment's raw header bytes: formatVersion 2, sealSize 4096. + const manifest = await readManifest(brain) + expect(manifest.tailSegment).toBeTruthy() + expect(manifest.brainId, 'the brain id was minted into the manifest').toBeTruthy() + const bytes = segmentBytes(dir, manifest.tailSegment!) + const header = parseSegmentHeader(bytes.subarray(0, SEGMENT_HEADER_BYTES)) + expect(header.formatVersion).toBe(FACT_LOG_FORMAT_V2) + expect(header.sealSize).toBe(4096) + + // Genesis is the FIRST record of the FIRST fact — and appears exactly once. + const group = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES), { expectedIdSpaceWidth: 64 }) + expect(group.facts.length).toBeGreaterThanOrEqual(2) + const firstRecord = group.facts[0].records[0] + expect(firstRecord.type).toBe('log.genesis') + const genesis = firstRecord as LogGenesisRecord + expect(genesis.idSpaceWidth).toBe(64) + expect(genesis.brainId).toBe(manifest.brainId) + const genesisCount = group.facts + .flatMap((f) => f.records) + .filter((r) => r.type === 'log.genesis').length + expect(genesisCount).toBe(1) + + // Shape parity + reconstruction fidelity: REOPEN (so the tail decodes + // from disk, not from the in-session originals) and compare each add's + // CommitFact op against canonical byte truth — metadata leg (bigint + // timestamps normalized back to numbers) AND the reconstructed vector + // wrapper must equal what readNounRaw returns, exactly as a v1 log's + // byte-faithful capture would. + await (brain as unknown as { close: () => Promise }).close() + brains.splice(brains.indexOf(brain), 1) + const reopened = track(await openBrain(dir)) + const facts = await allFacts(reopened) + const gens = facts.map((f) => f.generation) + expect([...gens].sort((a, b) => a - b)).toEqual(gens) + expect(new Set(gens).size).toBe(gens.length) + + const logGens = new Set( + ((await (reopened as unknown as { transactionLog(): Promise> }).transactionLog()) ?? []).map( + (e) => e.generation + ) + ) + for (const g of gens) expect(logGens.has(g), `generation ${g} is a real commit`).toBe(true) + + for (const id of [idA, idB]) { + const fact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null)) + expect(fact, `the add fact for ${id} survives the reopen`).toBeDefined() + const op = fact!.ops.find((o) => o.id === id)! + expect(op.kind).toBe('noun') + const canonical = await internals(reopened).storage.readNounRaw(id) + expect(op.record!.metadata).toStrictEqual(canonical.metadata) + expect(op.record!.vector).toStrictEqual(canonical.vector) + } + }) + + it('(b) MIXED LOG: an existing v1 segment stays readable forever beside the v2 tail (cutover by rotation, v1 bytes untouched)', async () => { + // ROUTE: a REAL v1 segment is written by the v1 writer itself — the live + // FactLog with its minter stripped (the exact pre-cutover code path, + // still shipped for minter-less configurations) — then the minter is + // restored mid-session and the next append performs the cutover + // rotation. Stronger than hand-crafted bytes: both formats come from + // their real writers, on one log. + const dir = trackDir() + const brain = track(await openBrain(dir)) + const log = factLogOf(brain) + const minter = log.intMinter + expect(minter, 'the brain wired the int minter at init').toBeTruthy() + + log.intMinter = null // the pre-cutover writer + const idOld1 = uid('v1-old-1') + const idOld2 = uid('v1-old-2') + await brain.add({ id: idOld1, data: 'old one', type: NounType.Document, vector: vec(3), metadata: { era: 'v1' } }) + await brain.add({ id: idOld2, data: 'old two', type: NounType.Document, vector: vec(4), metadata: { era: 'v1' } }) + await brain.flush() + + const before = await readManifest(brain) + expect(before.segments).toHaveLength(0) + const v1TailFile = before.tailSegment! + const v1Bytes = segmentBytes(dir, v1TailFile) + expect(parseSegmentHeader(v1Bytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe( + FACT_LOG_FORMAT_V1 + ) + + log.intMinter = minter // the cutover lands mid-session + const idNew = uid('v2-new') + await brain.add({ id: idNew, data: 'new era', type: NounType.Document, vector: vec(5), metadata: { era: 'v2' } }) + await brain.flush() + + // The v1 tail was SEALED (bytes untouched), the new tail is v2. + const after = await readManifest(brain) + expect(after.segments.map((s) => s.file)).toContain(v1TailFile) + expect(after.tailSegment).not.toBe(v1TailFile) + const sealedBytes = segmentBytes(dir, v1TailFile) + expect(parseSegmentHeader(sealedBytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe( + FACT_LOG_FORMAT_V1 + ) + expect( + Buffer.compare(Buffer.from(sealedBytes), Buffer.from(v1Bytes)), + 'the sealed v1 segment is byte-identical — never rewritten' + ).toBe(0) + const tailBytes = segmentBytes(dir, after.tailSegment!) + expect(parseSegmentHeader(tailBytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe( + FACT_LOG_FORMAT_V2 + ) + // NOT a brand-new log: no genesis on a rotated-in v2 tail. + const tailGroup = decodeGroupV2(tailBytes.subarray(SEGMENT_HEADER_BYTES), { + expectedIdSpaceWidth: 64 + }) + expect( + tailGroup.facts.flatMap((f) => f.records).some((r) => r.type === 'log.genesis') + ).toBe(false) + + // One scan spans both formats, shape-identically, in generation order. + const liveFacts = await allFacts(brain) + const liveGens = liveFacts.map((f) => f.generation) + expect([...liveGens].sort((a, b) => a - b)).toEqual(liveGens) + for (const id of [idOld1, idOld2, idNew]) { + const fact = liveFacts.find((f) => f.ops.some((op) => op.id === id)) + expect(fact, `fact for ${id} is scannable`).toBeDefined() + const op = fact!.ops.find((o) => o.id === id)! + expect(op.kind).toBe('noun') + expect(op.record).not.toBeNull() + } + + // The MIXED log survives a reopen and keeps appending (v2 tail). + await (brain as unknown as { close: () => Promise }).close() + brains.splice(brains.indexOf(brain), 1) + const reopened = track(await openBrain(dir)) + const reFacts = await allFacts(reopened) + expect(reFacts.map((f) => f.generation)).toEqual(liveGens) + // The v1 fact still reads exactly as the v1 decoder always read it. + // (Not compared byte-strict against canonical: the v1 CAPTURE has a + // known pre-existing wart — write-cache-warm objects carry + // undefined-valued engine keys that msgpack preserves as nil while the + // durable JSON drops them. v1 bytes are frozen; the v2 encoder + // sanitizes to durable truth instead — pinned in scenario (a).) + const oldOp = reFacts + .find((f) => f.ops.some((op) => op.id === idOld1))! + .ops.find((o) => o.id === idOld1)! + const canonicalOld = await internals(reopened).storage.readNounRaw(idOld1) + const oldMeta = oldOp.record!.metadata as Record + expect(oldMeta.noun).toBe('document') + expect((oldMeta.metadata as Record).era).toBe('v1') + const oldWrapper = oldOp.record!.vector as { id: string; vector: number[] } + const canonicalWrapper = canonicalOld.vector as { id: string; vector: number[] } + expect(oldWrapper.id).toBe(idOld1) + expect(oldWrapper.vector).toStrictEqual(canonicalWrapper.vector) + await reopened.add({ id: uid('post-reopen'), data: 'still writing', type: NounType.Document, vector: vec(6), metadata: {} }) + expect((await factGenerations(reopened)).length).toBe(liveGens.length + 1) + }) + + it('(c) MINT-AT-APPEND: after-image records carry the id mapper\'s EXACT int assignments — distinct, nonzero, reproducible', async () => { + const dir = trackDir() + const brain = track(await openBrain(dir)) + const idA = uid('mint-a') + const idB = uid('mint-b') + await brain.add({ id: idA, data: 'mint one', type: NounType.Document, vector: vec(7), metadata: { m: 1 } }) + await brain.add({ id: idB, data: 'mint two', type: NounType.Document, vector: vec(8), metadata: { m: 2 } }) + await brain.flush() + + const manifest = await readManifest(brain) + const bytes = segmentBytes(dir, manifest.tailSegment!) + const group = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES), { expectedIdSpaceWidth: 64 }) + const afterImages = new Map() + for (const fact of group.facts) { + for (const record of fact.records) { + if (record.type === 'noun.afterImage') afterImages.set(record.id, record) + } + } + const recA = afterImages.get(idA) + const recB = afterImages.get(idB) + expect(recA, 'idA has a decoded after-image').toBeDefined() + expect(recB, 'idB has a decoded after-image').toBeDefined() + expect(recA!.entityInt).toBeGreaterThan(0n) + expect(recB!.entityInt).toBeGreaterThan(0n) + expect(recA!.entityInt).not.toBe(recB!.entityInt) + + // White-box: the ints on the wire ARE the metadata index mapper's + // assignments — the exact ints a mapper rebuild must reproduce. + const mapper = internals(brain).metadataIndex.getIdMapper() + expect(recA!.entityInt).toBe(BigInt(mapper.getInt(idA)!)) + expect(recB!.entityInt).toBe(BigInt(mapper.getInt(idB)!)) + }) + + it('(d) SEALS AT SYNC: every flush leaves the tail sector-aligned; pads are invisible to scans', async () => { + const dir = trackDir() + const brain = track(await openBrain(dir)) + await brain.add({ id: uid('seal-1'), data: 'one', type: NounType.Document, vector: vec(10), metadata: {} }) + await brain.flush() + + const manifest = await readManifest(brain) + const tailPath = path.join(dir, ...FACTS_DIR, manifest.tailSegment!) + const sizeAfterFirstFlush = fs.statSync(tailPath).size + expect(sizeAfterFirstFlush).toBeGreaterThan(0) + expect(sizeAfterFirstFlush % 4096, 'tail is sector-aligned after flush').toBe(0) + const countAfterFirstFlush = (await factGenerations(brain)).length + + for (let i = 0; i < 3; i++) { + await brain.add({ id: uid(`seal-more-${i}`), data: `more ${i}`, type: NounType.Document, vector: vec(11 + i), metadata: { i } }) + } + await brain.flush() + const sizeAfterSecondFlush = fs.statSync(tailPath).size + expect(sizeAfterSecondFlush).toBeGreaterThan(sizeAfterFirstFlush) + expect(sizeAfterSecondFlush % 4096, 'still aligned after more writes + flush').toBe(0) + + // Pads count toward bytes, never toward facts. + expect((await factGenerations(brain)).length).toBe(countAfterFirstFlush + 3) + }) + + it('(e) REPLAY COMPAT: the log-authority recovery path resurrects an acked write from a v2 tail after a crash-style abandon', async () => { + // The flip idiom from the log-authority suite: seed writes, baseline + // backfill LAST (the init-time VFS root never got a fact), flush, then + // the sanctioned guarded flip — the oracle goes green over an ALL-V2 + // log, which is itself the reproduction proof for the v2 record path. + const dir = mkdtempSync(join(tmpdir(), 'brainy-v2-cutover-')) + dirs.push(dir) + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + const open = async (): Promise => { + const b = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false, + silent: true, + dimensions: 384 + }) + await b.init() + return track(b) + } + + const brain = await open() + const kept = await brain.add({ data: 'alpha document', type: 'document', metadata: { n: 1 } }) + const removed = await brain.add({ data: 'beta document', type: 'document', metadata: { n: 2 } }) + await brain.update({ id: kept, metadata: { n: 10 } }) + await brain.remove(removed) + const root = await brain.get(VFS_ROOT) + expect(root, 'the VFS root exists').toBeTruthy() + await brain.update({ id: VFS_ROOT, metadata: root!.metadata }) // baseline backfill — final write + await brain.flush() + + const report = await (brain as unknown as { adoptLogAuthority(): Promise<{ verdict: string }> }).adoptLogAuthority() + expect(report.verdict, 'the oracle is green over a pure-v2 log').toBe('green') + + // An at-ack write: its v2 fact is fsynced (sector-sealed) at ack. + const survivor = await brain.add({ + data: 'survives power loss', + type: 'document', + metadata: { s: 1 } + }) + + // Crash-style abandon: RAM state gone, no flush, no close. + await abandonAsCrashed(brain) + + // Reopen: open() finds the acked fact ABOVE the manifest watermark in + // the v2 tail (peekFactsAbove → v2 decode) and REPLAYS it into + // canonical — an acked write is never lost. + const reopened = await open() + expect( + (reopened as unknown as { logAuthority(): { authority: string } }).logAuthority().authority + ).toBe('log') + const resurrected = await reopened.get(survivor) + expect(resurrected, 'the acked write survived the crash').toBeTruthy() + expect((resurrected as { metadata?: { s?: number } }).metadata?.s).toBe(1) + expect((await factGenerations(reopened)).length).toBeGreaterThan(0) + }) +}) diff --git a/tests/integration/log-authority.test.ts b/tests/integration/log-authority.test.ts index 14278cd1..e0984321 100644 --- a/tests/integration/log-authority.test.ts +++ b/tests/integration/log-authority.test.ts @@ -41,6 +41,7 @@ type BrainInternals = { saveNoun(n: unknown): Promise saveNounMetadata(id: string, m: Record): Promise getNounMetadata(id: string): Promise | null> + writeNounRaw(id: string, r: { metadata: null; vector: null }): Promise } } @@ -219,28 +220,21 @@ describe('log authority — the switch, the oracle, the guarded flip', () => { }) }) - it('THE FLIP REFUSES ON RED: names the oracle verdict and the cure, writes nothing, changes nothing', async () => { + it('THE FLIP REFUSES ON A LOG-AHEAD DIVERGENCE: the witness denies what the log claims — nothing written, nothing changed', async () => { + // Contract update (adoptLogAuthority's baseline backfill): curable + // divergences — pre-log records and witness drift — are re-committed + // and the flip proceeds; ONLY log-AHEAD divergences (the log claims + // state canonical denies) refuse, because no backfill can make the log + // un-claim a live row. This test stages exactly that incurable shape. const { brain } = await openBrain() - await seedWrites(brain) + const { kept } = await seedWrites(brain) await backfillBaseline(brain) await brain.flush() - // Age the brain: one canonical record the log never saw. - const legacyId = '00000000-0000-4000-8000-00000000a6ed' + // The log says `kept` is live; its canonical record vanishes behind the + // write path's back (log-live-canonical-absent — the witness wins). const storage = internals(brain).storage - await storage.saveNoun({ - id: legacyId, - vector: new Array(384).fill(0.01), - connections: new Map(), - level: 0 - }) - await storage.saveNounMetadata(legacyId, { - noun: 'document', - confidence: 0.5, - createdAt: 1700000000000, - updatedAt: 1700000000000, - _rev: 1 - }) + await storage.writeNounRaw(kept, { metadata: null, vector: null }) let error: Error | null = null try { @@ -248,9 +242,9 @@ describe('log authority — the switch, the oracle, the guarded flip', () => { } catch (err) { error = err as Error } - expect(error, 'the flip rejects on a red oracle').not.toBeNull() - expect(error!.message).toMatch(/oracle is RED/) - expect(error!.message).toMatch(/baseline backfill/) + expect(error, 'the flip rejects on a log-ahead divergence').not.toBeNull() + expect(error!.message).toMatch(/witness denies/) + expect(error!.message).toMatch(/log-live-canonical-absent/) // Nothing changed: authority still tree, no artifact, deferred durability. expect(brain.logAuthority().authority).toBe('tree') diff --git a/tests/unit/db/factLogFormat.test.ts b/tests/unit/db/factLogFormat.test.ts index ec1aedb2..0c507b41 100644 --- a/tests/unit/db/factLogFormat.test.ts +++ b/tests/unit/db/factLogFormat.test.ts @@ -3,7 +3,9 @@ * @description Fact-log format v2 (record envelope + sector seals) pinned at * the byte level: every record type round-trips field-exact (bigint ints, * bin16 uuids, float-exact vectors), headers read v1 AND v2, unknown record - * types/versions refuse loudly with the typed error, genesis width mismatches + * types/versions refuse loudly with the typed error, the reserved crypto + * envelope (cipherFlag/keyId — plaintext-only this release) refuses anything + * nonzero/non-nil with the same typed error, genesis width mismatches * refuse naming both widths, sealed groups align to the sector size with * invisible pads, vector refs are writer-enforced single-hop, and torn tails * truncate to the intact prefix at EVERY byte offset. This module is the @@ -11,7 +13,7 @@ * vectors here are frozen; a change that breaks them is a format change. */ import { describe, it, expect } from 'vitest' -import { encode } from '@msgpack/msgpack' +import { encode, decode } from '@msgpack/msgpack' import { encodeFactV2, decodeFact, @@ -20,10 +22,13 @@ import { parseSegmentHeader, sealGroup, framePayload, + encodePadFrame, + minPadFrameBytes, UnknownLogRecordError, GenesisWidthMismatchError, LOG_RECORD_TYPES, LOG_RECORD_VERSION, + LOG_RECORD_CIPHER_PLAINTEXT, FACT_LOG_FORMAT_V1, FACT_LOG_FORMAT_V2, SEGMENT_HEADER_BYTES, @@ -254,7 +259,7 @@ describe('fact-log format v2 — golden byte vectors (frozen contract)', () => { records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }] }) expect(hex(frame)).toBe( - '2b000000c19ad9ff95cf0000000000000003cf0000018bcfe5687b91930201' + + '2d00000048e4d43695cf0000000000000003cf0000018bcfe5687b9195020100c0' + 'c41000000000000040008000000000000042c0c0' ) }) @@ -370,11 +375,51 @@ describe('fact-log format v2 — decoder law (typed refusals, never skip)', () = }) it('a fact mixing known and unknown records still refuses (no partial reads)', () => { - const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, uuidBytes(UUID(1))] + const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))] const payload = encode([1, 1, [known, [200, 1]], null, null]) expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError) }) + it('a nonzero cipherFlag refuses with the typed error — encrypted records need a newer reader', () => { + const payload = encode( + [1, 1, [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 1, null, uuidBytes(UUID(1))]], null, null] + ) + try { + decodeFact(payload, 2) + expect.unreachable('decode must throw') + } catch (error) { + const typed = error as UnknownLogRecordError + expect(typed).toBeInstanceOf(UnknownLogRecordError) + expect(typed.recordType).toBe(LOG_RECORD_TYPES.NOUN_TOMBSTONE) + expect(typed.recordVersion).toBe(1) + expect(typed.message).toMatch(/cipherFlag 1/) + expect(typed.message).toMatch(/encrypted records need a newer reader/) + } + }) + + it('a non-nil keyId refuses the same way, even with cipherFlag 0', () => { + const payload = encode( + [ + 1, + 1, + [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, uuidBytes(UUID(9)), uuidBytes(UUID(1))]], + null, + null + ] + ) + expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError) + expect(() => decodeFact(payload, 2)).toThrow(/encrypted records need a newer reader/) + }) + + it('the encoder always writes the plaintext envelope: cipherFlag 0, keyId nil', () => { + const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) }))) + const raw = decode(payload) as unknown[] + const record = (raw[2] as unknown[][])[0] + expect(record[2]).toBe(LOG_RECORD_CIPHER_PLAINTEXT) + expect(record[3]).toBeNull() + expect(LOG_RECORD_CIPHER_PLAINTEXT).toBe(0) + }) + it('an unknown segment format version has no decode path', () => { const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) }))) expect(() => decodeFact(payload, 3)).toThrow(/reads 1 and 2/) @@ -424,8 +469,8 @@ describe('fact-log format v2 — log.genesis width law', () => { 1, 1, [ - [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, uuidBytes(UUID(1))], - [LOG_RECORD_TYPES.LOG_GENESIS, 1, 64, uuidBytes(UUID(9)), 1] + [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))], + [LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 64, uuidBytes(UUID(9)), 1] ], null, null @@ -434,7 +479,9 @@ describe('fact-log format v2 — log.genesis width law', () => { }) it('an invalid genesis width on the wire is malformed, not a mismatch', () => { - const crafted = encode([1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 48, uuidBytes(UUID(9)), 1]], null, null]) + const crafted = encode( + [1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 48, uuidBytes(UUID(9)), 1]], null, null] + ) expect(() => decodeFact(crafted, 2)).toThrow(/32 or 64/) }) }) @@ -504,7 +551,7 @@ describe('fact-log format v2 — vector legs (single-hop law)', () => { }) expect(() => encodeFactV2(bad)).toThrow(/INLINE/) const craftedRef = encode( - [1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, uuidBytes(UUID(7)), ['ref', 5]]], null, null] + [1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, 0, null, uuidBytes(UUID(7)), ['ref', 5]]], null, null] ) expect(() => decodeFact(craftedRef, 2)).toThrow(/INLINE/) }) @@ -571,16 +618,30 @@ describe('fact-log format v2 — sector seals', () => { timestamp: 1_700_000_000_123, records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }] }) - const sealed = sealGroup([tomb], 64) // 51 bytes → gap 13 → overshoot → 77-byte pad + const sealed = sealGroup([tomb], 64) // 53 bytes → gap 11 → overshoot → 75-byte pad expect(sealed.length).toBe(128) expect(hex(sealed.subarray(tomb.length))).toBe( - // frame prefix + [0, 0, [[0, 1, bin8(42 zero bytes)]], nil, nil] - '450000009463044d95cf0000000000000000cf000000000000000091930001c42a' + - '0'.repeat(84) + + // frame prefix + [0, 0, [[0, 1, bin8(40 zero bytes)]], nil, nil] + '4300000088b4c8fa95cf0000000000000000cf000000000000000091930001c428' + + '0'.repeat(80) + 'c0c0' ) }) + it('encodePadFrame builds exact-size pads for streaming writers; refuses sub-minimum sizes', () => { + // Pads are envelope-exempt (skipped wholesale), so the smallest pad frame + // is byte-stable across the crypto-envelope change. + expect(minPadFrameBytes()).toBe(33) + for (const size of [minPadFrameBytes(), 64, 4096]) { + const pad = encodePadFrame(size) + expect(pad.length).toBe(size) + const { facts: decoded, validBytes } = decodeGroupV2(pad) + expect(decoded).toEqual([]) // invisible to readers + expect(validBytes).toBe(size) + } + expect(() => encodePadFrame(minPadFrameBytes() - 1)).toThrow(/at least/) + }) + it('sealGroup refuses garbage: empty groups, malformed frames, bad seal sizes', () => { expect(() => sealGroup([], 4096)).toThrow(/at least one frame/) expect(() => sealGroup([new Uint8Array([1, 2, 3])], 4096)).toThrow(/not a well-formed frame/) @@ -620,10 +681,12 @@ describe('fact-log format v2 — torn-tail discipline', () => { describe('fact-log format v2 — writer refusals (loud, never silent)', () => { const tombstone = (g: number): CommitFactV2 => factOf(g, { type: 'noun.tombstone', id: UUID(g) }) - it('refuses empty records, generation 0, and a second batch.meta', () => { - expect(() => encodeFactV2({ generation: 1, timestamp: 1, records: [] })).toThrow( - /at least one record/ - ) + it('accepts empty records (an all-deduped batch is a real generation); refuses generation 0 and a second batch.meta', () => { + // Contract change with the live cutover: v1 always encoded op-less + // commits (a batch whose relates dedupe away still mints a generation); + // v2 must not fork commit semantics — empty records round-trip. + const empty = decodeFact(framePayload(encodeFactV2({ generation: 1, timestamp: 1, records: [] })), 2) + expect(empty.records).toEqual([]) expect(() => encodeFactV2({ ...tombstone(1), generation: 0 })).toThrow(/positive integer/) expect(() => encodeFactV2({