feat(log): v2 is the LIVE write format — envelope records with minted ints, genesis, sector seals; v1 readable forever

The cutover: new tail segments write format v2 (per-record [type, version,
cipherFlag, keyId] envelope; noun/verb after-images carry dense ints
MINTED AT APPEND from the id mapper — a rebuilt mapper reproduces
assignments exactly; log.genesis opens every new log with the id-space
width + a minted brain id; sync() seals to the header-declared sector
boundary with reader-invisible pad frames). Existing v1 segments are
never rewritten — per-segment decoder dispatch reads both formats and v2
facts map to the exact CommitFact shape all consumers already read.
Cutover on a live v1 log: an empty v1 tail re-heads in place; a non-empty
one is sealed by rotation, byte-identical. Records reserve the encryption
fields (cipherFlag 0 / keyId nil are the only legal values; anything else
refuses typed naming the needed newer reader) — crypto-ready with no
future bump on the compat surface. Empty-records facts are legal (an
all-deduped batch is a real generation — v1 semantics preserved; the
refusal there tore a column-store flush mid-commit in the full suite, the
consistency guard caught it loudly, and the root is fixed).

Golden byte vectors pinned for the second (native) reader implementation.
Pins: cutover 5/5 · codec 54 · kill-matrix stays 11/11.
This commit is contained in:
David Snelling 2026-08-10 10:55:11 -07:00
parent 73eb88d481
commit 26c6025158
6 changed files with 1372 additions and 135 deletions

View file

@ -26,9 +26,21 @@
* position 2 is `records`, not v1's `ops`)
*
* fact := [ generation:u64, timestamp:u64, records, meta|nil, blobHashes|nil ]
* record := [ recordType:u8, recordVersion:u8, ...type-specific fields ]
* record := [ recordType:u8, recordVersion:u8, cipherFlag:u8, keyId:bin16|nil,
* ...type-specific fields ]
*
* Record type registry (all recordVersion = 1):
* `cipherFlag`/`keyId` are RESERVED crypto envelope fields: `0`/`nil` (a
* plaintext record) is the ONLY legal combination this release writes or
* reads. Any nonzero cipherFlag or non-nil keyId refuses with the typed
* {@link UnknownLogRecordError} ("encrypted records need a newer reader")
* so record-level encryption can land later without a format-version bump on
* the one compat surface. No crypto logic exists here; the bytes are reserved
* only. Pad records (type 0) are exempt: they are skipped WHOLESALE as
* length-only filler, so their fields beyond [type, version] are never
* inspected (this keeps pad frames byte-stable across the envelope change).
*
* Record type registry (all recordVersion = 1; type-specific fields listed
* every record carries the 4-field envelope above first):
*
* 0 pad [] length-only filler; readers SKIP; crc-covered
* 1 noun.afterImage [id bin16, entityInt u64, metadata, vectorLeg]
@ -109,6 +121,13 @@ export const DEFAULT_SEAL_SIZE = 4096
/** The record version this reader knows (all registry types are version 1). */
export const LOG_RECORD_VERSION = 1
/**
* The only legal `cipherFlag` value this release: plaintext. The encoder
* always writes it (with a nil keyId); the decoder refuses anything else
* with {@link UnknownLogRecordError} encrypted records need a newer reader.
*/
export const LOG_RECORD_CIPHER_PLAINTEXT = 0
/** The v2 record-type registry — wire codes for every record type. */
export const LOG_RECORD_TYPES = {
PAD: 0,
@ -648,22 +667,31 @@ function decodeVectorLeg(wire: unknown, context: string): VectorLeg {
// Record encode/decode
// ---------------------------------------------------------------------------
/** Encode one record into its positional wire array. */
/**
* Encode one record into its positional wire array. Every record leads with
* the 4-field envelope [type, version, cipherFlag, keyId]; this release
* writes cipherFlag {@link LOG_RECORD_CIPHER_PLAINTEXT} and a nil keyId
* always (the fields are crypto-RESERVED, carrying no logic yet).
*/
function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefined): unknown[] {
const T = LOG_RECORD_TYPES
const V = LOG_RECORD_VERSION
const C = LOG_RECORD_CIPHER_PLAINTEXT
const K = null // keyId: nil until record-level encryption exists
switch (record.type) {
case 'noun.afterImage':
return [
T.NOUN_AFTER_IMAGE,
V,
C,
K,
uuidToBytes(record.id),
toWireU64(record.entityInt, 'entityInt'),
record.metadata ?? null,
encodeVectorLeg(record.vectorLeg, options, `noun.afterImage ${record.id}`)
]
case 'noun.tombstone':
return [T.NOUN_TOMBSTONE, V, uuidToBytes(record.id)]
return [T.NOUN_TOMBSTONE, V, C, K, uuidToBytes(record.id)]
case 'verb.afterImage': {
if (typeof record.verb !== 'string' || record.verb.length === 0) {
throw new Error(`fact log v2: verb.afterImage ${record.id} needs a non-empty verb name`)
@ -671,6 +699,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
return [
T.VERB_AFTER_IMAGE,
V,
C,
K,
uuidToBytes(record.id),
toWireU64(record.verbInt, 'verbInt'),
record.metadata ?? null,
@ -683,16 +713,18 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
]
}
case 'verb.tombstone':
return [T.VERB_TOMBSTONE, V, uuidToBytes(record.id)]
return [T.VERB_TOMBSTONE, V, C, K, uuidToBytes(record.id)]
case 'batch.meta':
if (!isPlainMap(record.meta)) {
throw new Error('fact log v2: batch.meta requires a map')
}
return [T.BATCH_META, V, record.meta]
return [T.BATCH_META, V, C, K, record.meta]
case 'embed.pending':
return [
T.EMBED_PENDING,
V,
C,
K,
uuidToBytes(record.id),
toWireU64(record.enqueuedAt, 'enqueuedAt')
]
@ -703,7 +735,7 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
`refs and nil are not allowed here`
)
}
return [T.EMBED_LANDED, V, uuidToBytes(record.id), record.vector]
return [T.EMBED_LANDED, V, C, K, uuidToBytes(record.id), record.vector]
}
case 'blob.manifest': {
if (typeof record.mimeType !== 'string') {
@ -715,6 +747,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
return [
T.BLOB_MANIFEST,
V,
C,
K,
hashToBytes(record.hash),
toWireU64(record.size, 'blob size'),
record.mimeType,
@ -725,7 +759,7 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
if (!isPlainMap(record.note)) {
throw new Error('fact log v2: projection.note requires a map')
}
return [T.PROJECTION_NOTE, V, record.note]
return [T.PROJECTION_NOTE, V, C, K, record.note]
case 'bootstrap.baseline': {
if (record.kind !== 'noun' && record.kind !== 'verb') {
throw new Error(`fact log v2: bootstrap.baseline kind must be 'noun' or 'verb'`)
@ -733,6 +767,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
return [
T.BOOTSTRAP_BASELINE,
V,
C,
K,
uuidToBytes(record.id),
record.kind === 'noun' ? 0 : 1,
record.metadata ?? null,
@ -748,6 +784,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
return [
T.LOG_GENESIS,
V,
C,
K,
record.idSpaceWidth,
uuidToBytes(record.brainId),
toWireU64(record.createdAt, 'createdAt')
@ -763,35 +801,39 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
}
}
/** Exact wire arity per record type (envelope of 2 + type-specific fields). */
/** Exact wire arity per record type (envelope of 4 + type-specific fields). */
const RECORD_ARITY: Record<number, number> = {
[LOG_RECORD_TYPES.NOUN_AFTER_IMAGE]: 6,
[LOG_RECORD_TYPES.NOUN_TOMBSTONE]: 3,
[LOG_RECORD_TYPES.VERB_AFTER_IMAGE]: 11,
[LOG_RECORD_TYPES.VERB_TOMBSTONE]: 3,
[LOG_RECORD_TYPES.BATCH_META]: 3,
[LOG_RECORD_TYPES.EMBED_PENDING]: 4,
[LOG_RECORD_TYPES.EMBED_LANDED]: 4,
[LOG_RECORD_TYPES.BLOB_MANIFEST]: 6,
[LOG_RECORD_TYPES.PROJECTION_NOTE]: 3,
[LOG_RECORD_TYPES.BOOTSTRAP_BASELINE]: 6,
[LOG_RECORD_TYPES.LOG_GENESIS]: 5
[LOG_RECORD_TYPES.NOUN_AFTER_IMAGE]: 8,
[LOG_RECORD_TYPES.NOUN_TOMBSTONE]: 5,
[LOG_RECORD_TYPES.VERB_AFTER_IMAGE]: 13,
[LOG_RECORD_TYPES.VERB_TOMBSTONE]: 5,
[LOG_RECORD_TYPES.BATCH_META]: 5,
[LOG_RECORD_TYPES.EMBED_PENDING]: 6,
[LOG_RECORD_TYPES.EMBED_LANDED]: 6,
[LOG_RECORD_TYPES.BLOB_MANIFEST]: 8,
[LOG_RECORD_TYPES.PROJECTION_NOTE]: 5,
[LOG_RECORD_TYPES.BOOTSTRAP_BASELINE]: 8,
[LOG_RECORD_TYPES.LOG_GENESIS]: 7
}
/**
* Decode one wire record. Returns `null` for pads (skipped by definition).
* Unknown type / newer version throw {@link UnknownLogRecordError} never
* skip-and-continue.
* skip-and-continue. The reserved crypto envelope is verified BEFORE the
* arity check (an encrypted record's field layout is a newer reader's
* business, not a malformed-record error): any nonzero cipherFlag or non-nil
* keyId refuses with the same typed error class.
*/
function decodeRecord(raw: unknown): LogRecord | null {
if (!Array.isArray(raw) || raw.length < 2) {
throw new Error('fact log v2: malformed record envelope (need [type, version, ...])')
throw new Error('fact log v2: malformed record envelope (need [type, version, cipherFlag, keyId, ...])')
}
const recordType = wireToU8(raw[0], 'recordType')
const recordVersion = wireToU8(raw[1], 'recordVersion')
if (recordType === LOG_RECORD_TYPES.PAD) {
// Length-only filler: skipped wholesale, filler fields never inspected.
// Length-only filler: skipped wholesale, filler fields never inspected
// (pads therefore carry no crypto envelope — by definition, not omission).
return null
}
const arity = RECORD_ARITY[recordType]
@ -814,6 +856,20 @@ function decodeRecord(raw: unknown): LogRecord | null {
if (recordVersion !== LOG_RECORD_VERSION) {
throw new Error(`fact log v2: record type ${recordType} has invalid record version ${recordVersion}`)
}
if (raw.length < 4) {
throw new Error('fact log v2: malformed record envelope (need [type, version, cipherFlag, keyId, ...])')
}
const cipherFlag = wireToU8(raw[2], 'cipherFlag')
const keyId = raw[3]
if (cipherFlag !== LOG_RECORD_CIPHER_PLAINTEXT || (keyId !== null && keyId !== undefined)) {
throw new UnknownLogRecordError(
recordType,
recordVersion,
`fact log v2: record type ${recordType} carries cipherFlag ${cipherFlag}` +
`${keyId !== null && keyId !== undefined ? ' and a keyId' : ''}` +
`encrypted records need a newer reader`
)
}
if (raw.length !== arity) {
throw new Error(
`fact log v2: record type ${recordType} expects ${arity} wire fields; got ${raw.length}`
@ -824,94 +880,94 @@ function decodeRecord(raw: unknown): LogRecord | null {
case LOG_RECORD_TYPES.NOUN_AFTER_IMAGE:
return {
type: 'noun.afterImage',
id: bytesToUuid(raw[2], 'noun.afterImage id'),
entityInt: wireToBigint(raw[3], 'entityInt'),
metadata: raw[4] ?? null,
vectorLeg: decodeVectorLeg(raw[5], 'noun.afterImage')
id: bytesToUuid(raw[4], 'noun.afterImage id'),
entityInt: wireToBigint(raw[5], 'entityInt'),
metadata: raw[6] ?? null,
vectorLeg: decodeVectorLeg(raw[7], 'noun.afterImage')
}
case LOG_RECORD_TYPES.NOUN_TOMBSTONE:
return { type: 'noun.tombstone', id: bytesToUuid(raw[2], 'noun.tombstone id') }
return { type: 'noun.tombstone', id: bytesToUuid(raw[4], 'noun.tombstone id') }
case LOG_RECORD_TYPES.VERB_AFTER_IMAGE: {
if (typeof raw[6] !== 'string') {
if (typeof raw[8] !== 'string') {
throw new Error('fact log v2: verb.afterImage verb name is not a string')
}
return {
type: 'verb.afterImage',
id: bytesToUuid(raw[2], 'verb.afterImage id'),
verbInt: wireToBigint(raw[3], 'verbInt'),
metadata: raw[4] ?? null,
vectorLeg: decodeVectorLeg(raw[5], 'verb.afterImage'),
verb: raw[6],
sourceId: bytesToUuid(raw[7], 'verb.afterImage sourceId'),
sourceInt: wireToBigint(raw[8], 'sourceInt'),
targetId: bytesToUuid(raw[9], 'verb.afterImage targetId'),
targetInt: wireToBigint(raw[10], 'targetInt')
id: bytesToUuid(raw[4], 'verb.afterImage id'),
verbInt: wireToBigint(raw[5], 'verbInt'),
metadata: raw[6] ?? null,
vectorLeg: decodeVectorLeg(raw[7], 'verb.afterImage'),
verb: raw[8],
sourceId: bytesToUuid(raw[9], 'verb.afterImage sourceId'),
sourceInt: wireToBigint(raw[10], 'sourceInt'),
targetId: bytesToUuid(raw[11], 'verb.afterImage targetId'),
targetInt: wireToBigint(raw[12], 'targetInt')
}
}
case LOG_RECORD_TYPES.VERB_TOMBSTONE:
return { type: 'verb.tombstone', id: bytesToUuid(raw[2], 'verb.tombstone id') }
return { type: 'verb.tombstone', id: bytesToUuid(raw[4], 'verb.tombstone id') }
case LOG_RECORD_TYPES.BATCH_META: {
if (!isPlainMap(raw[2])) throw new Error('fact log v2: batch.meta payload is not a map')
return { type: 'batch.meta', meta: raw[2] }
if (!isPlainMap(raw[4])) throw new Error('fact log v2: batch.meta payload is not a map')
return { type: 'batch.meta', meta: raw[4] }
}
case LOG_RECORD_TYPES.EMBED_PENDING:
return {
type: 'embed.pending',
id: bytesToUuid(raw[2], 'embed.pending id'),
enqueuedAt: wireToNumber(raw[3], 'enqueuedAt')
id: bytesToUuid(raw[4], 'embed.pending id'),
enqueuedAt: wireToNumber(raw[5], 'enqueuedAt')
}
case LOG_RECORD_TYPES.EMBED_LANDED: {
const leg = decodeVectorLeg(raw[3], 'embed.landed')
const leg = decodeVectorLeg(raw[5], 'embed.landed')
if (!Array.isArray(leg)) {
throw new Error(
'fact log v2: embed.landed must carry an INLINE float vector — refs and nil are not allowed here'
)
}
return { type: 'embed.landed', id: bytesToUuid(raw[2], 'embed.landed id'), vector: leg }
return { type: 'embed.landed', id: bytesToUuid(raw[4], 'embed.landed id'), vector: leg }
}
case LOG_RECORD_TYPES.BLOB_MANIFEST: {
if (typeof raw[4] !== 'string') {
if (typeof raw[6] !== 'string') {
throw new Error('fact log v2: blob.manifest mimeType is not a string')
}
const refOp = wireToU8(raw[5], 'refOp')
const refOp = wireToU8(raw[7], 'refOp')
if (refOp !== 0 && refOp !== 1) {
throw new Error(`fact log v2: blob.manifest refOp must be 0 (add) or 1 (release); got ${refOp}`)
}
return {
type: 'blob.manifest',
hash: bytesToHash(raw[2]),
size: wireToNumber(raw[3], 'blob size'),
mimeType: raw[4],
hash: bytesToHash(raw[4]),
size: wireToNumber(raw[5], 'blob size'),
mimeType: raw[6],
refOp: refOp === 0 ? 'add' : 'release'
}
}
case LOG_RECORD_TYPES.PROJECTION_NOTE: {
if (!isPlainMap(raw[2])) throw new Error('fact log v2: projection.note payload is not a map')
return { type: 'projection.note', note: raw[2] }
if (!isPlainMap(raw[4])) throw new Error('fact log v2: projection.note payload is not a map')
return { type: 'projection.note', note: raw[4] }
}
case LOG_RECORD_TYPES.BOOTSTRAP_BASELINE: {
const kind = wireToU8(raw[3], 'bootstrap.baseline kind')
const kind = wireToU8(raw[5], 'bootstrap.baseline kind')
if (kind !== 0 && kind !== 1) {
throw new Error(`fact log v2: bootstrap.baseline kind must be 0 (noun) or 1 (verb); got ${kind}`)
}
return {
type: 'bootstrap.baseline',
id: bytesToUuid(raw[2], 'bootstrap.baseline id'),
id: bytesToUuid(raw[4], 'bootstrap.baseline id'),
kind: kind === 0 ? 'noun' : 'verb',
metadata: raw[4] ?? null,
vectorLeg: decodeVectorLeg(raw[5], 'bootstrap.baseline')
metadata: raw[6] ?? null,
vectorLeg: decodeVectorLeg(raw[7], 'bootstrap.baseline')
}
}
case LOG_RECORD_TYPES.LOG_GENESIS: {
const width = wireToU8(raw[2], 'idSpaceWidth')
const width = wireToU8(raw[4], 'idSpaceWidth')
if (width !== 32 && width !== 64) {
throw new Error(`fact log v2: log.genesis idSpaceWidth must be 32 or 64; got ${width}`)
}
return {
type: 'log.genesis',
idSpaceWidth: width,
brainId: bytesToUuid(raw[3], 'log.genesis brainId'),
createdAt: wireToNumber(raw[4], 'createdAt')
brainId: bytesToUuid(raw[5], 'log.genesis brainId'),
createdAt: wireToNumber(raw[6], 'createdAt')
}
}
default:
@ -944,8 +1000,12 @@ export function encodeFactV2(fact: CommitFactV2, options?: EncodeFactV2Options):
if (!Number.isSafeInteger(fact.timestamp) || fact.timestamp < 0) {
throw new Error(`fact log v2: timestamp must be a non-negative integer; got ${fact.timestamp}`)
}
if (!Array.isArray(fact.records) || fact.records.length === 0) {
throw new Error('fact log v2: a fact must carry at least one record')
// records MAY be empty: a committed generation whose ops all collapsed
// (e.g. a batch whose relates deduped to no-ops) is still a real
// generation — v1 encoded empty ops the same way; refusing here would
// fork the two formats' commit semantics.
if (!Array.isArray(fact.records)) {
throw new Error('fact log v2: records must be an array')
}
if (fact.meta !== undefined && !isPlainMap(fact.meta)) {
throw new Error('fact log v2: fact meta must be a map when present')
@ -1092,9 +1152,15 @@ function decodeFactV2(payload: Uint8Array, options?: DecodeFactV2Options): Commi
// Sector seals
// ---------------------------------------------------------------------------
/** Smallest constructible pad frame (envelope + bare pad record), memoized. */
/**
* Smallest constructible pad frame in bytes (frame prefix + the bare pad
* record fact), memoized. Exported for streaming writers that pad an
* append-only tail to a seal boundary: a gap smaller than this cannot hold
* any frame, so the writer pads through one extra sector (the same rule
* {@link sealGroup} applies).
*/
let minPadFrameBytesMemo: number | null = null
function minPadFrameBytes(): number {
export function minPadFrameBytes(): number {
if (minPadFrameBytesMemo === null) {
minPadFrameBytesMemo =
FRAME_PREFIX_BYTES +
@ -1145,6 +1211,25 @@ function buildPadFrame(totalBytes: number): Uint8Array {
return buildFrame(payload)
}
/**
* Build a pad frame of EXACTLY `totalBytes` the streaming-append counterpart
* of {@link sealGroup} for writers that append pads directly to a live tail
* instead of sealing an in-memory group. Refuses sizes smaller than the
* smallest constructible pad frame ({@link minPadFrameBytes}); readers skip
* the result by definition (a type-0 record is length-only filler).
*
* @param totalBytes - The exact frame size to construct (prefix included).
* @returns The complete pad frame bytes.
*/
export function encodePadFrame(totalBytes: number): Uint8Array {
if (!Number.isInteger(totalBytes) || totalBytes < minPadFrameBytes()) {
throw new Error(
`fact log v2: a pad frame must be at least ${minPadFrameBytes()} bytes; got ${totalBytes}`
)
}
return buildPadFrame(totalBytes)
}
/**
* Seal a group of frames to a sector boundary: concatenate the frames and pad
* to the next `sealSize` multiple with ONE pad frame. An already-aligned