soulcraftlabs/open-brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 341 Packages Wiki Activity Actions 5

feat(log): v2 is the LIVE write format — envelope records with minted ints, genesis, sector seals; v1 readable forever

Browse source 
The cutover: new tail segments write format v2 (per-record [type, version,
cipherFlag, keyId] envelope; noun/verb after-images carry dense ints
MINTED AT APPEND from the id mapper — a rebuilt mapper reproduces
assignments exactly; log.genesis opens every new log with the id-space
width + a minted brain id; sync() seals to the header-declared sector
boundary with reader-invisible pad frames). Existing v1 segments are
never rewritten — per-segment decoder dispatch reads both formats and v2
facts map to the exact CommitFact shape all consumers already read.
Cutover on a live v1 log: an empty v1 tail re-heads in place; a non-empty
one is sealed by rotation, byte-identical. Records reserve the encryption
fields (cipherFlag 0 / keyId nil are the only legal values; anything else
refuses typed naming the needed newer reader) — crypto-ready with no
future bump on the compat surface. Empty-records facts are legal (an
all-deduped batch is a real generation — v1 semantics preserved; the
refusal there tore a column-store flush mid-commit in the full suite, the
consistency guard caught it loudly, and the root is fixed).

Golden byte vectors pinned for the second (native) reader implementation.
Pins: cutover 5/5 · codec 54 · kill-matrix stays 11/11.
This commit is contained in:
David Snelling 2026-08-10 10:55:11 -07:00
parent 73eb88d481
commit 26c6025158
6 changed files with 1372 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

95
tests/unit/db/factLogFormat.test.ts
View file

@ -3,7 +3,9 @@
* @description Fact-log format v2 (record envelope + sector seals) pinned at
* the byte level: every record type round-trips field-exact (bigint ints,
* bin16 uuids, float-exact vectors), headers read v1 AND v2, unknown record
* types/versions refuse loudly with the typed error, genesis width mismatches
* types/versions refuse loudly with the typed error, the reserved crypto
* envelope (cipherFlag/keyId — plaintext-only this release) refuses anything
* nonzero/non-nil with the same typed error, genesis width mismatches
* refuse naming both widths, sealed groups align to the sector size with
* invisible pads, vector refs are writer-enforced single-hop, and torn tails
* truncate to the intact prefix at EVERY byte offset. This module is the
@ -11,7 +13,7 @@
* vectors here are frozen; a change that breaks them is a format change.
*/
import { describe, it, expect } from 'vitest'
import { encode } from '@msgpack/msgpack'
import { encode, decode } from '@msgpack/msgpack'
import {
encodeFactV2,
decodeFact,
@ -20,10 +22,13 @@ import {
parseSegmentHeader,
sealGroup,
framePayload,
encodePadFrame,
minPadFrameBytes,
UnknownLogRecordError,
GenesisWidthMismatchError,
LOG_RECORD_TYPES,
LOG_RECORD_VERSION,
LOG_RECORD_CIPHER_PLAINTEXT,
FACT_LOG_FORMAT_V1,
FACT_LOG_FORMAT_V2,
SEGMENT_HEADER_BYTES,
@ -254,7 +259,7 @@ describe('fact-log format v2 — golden byte vectors (frozen contract)', () => {
records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }]
})
expect(hex(frame)).toBe(
'2b000000c19ad9ff95cf0000000000000003cf0000018bcfe5687b91930201' +
'2d00000048e4d43695cf0000000000000003cf0000018bcfe5687b9195020100c0' +
'c41000000000000040008000000000000042c0c0'
)
})
@ -370,11 +375,51 @@ describe('fact-log format v2 — decoder law (typed refusals, never skip)', () =
})
it('a fact mixing known and unknown records still refuses (no partial reads)', () => {
const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, uuidBytes(UUID(1))]
const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))]
const payload = encode([1, 1, [known, [200, 1]], null, null])
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
})
it('a nonzero cipherFlag refuses with the typed error — encrypted records need a newer reader', () => {
const payload = encode(
[1, 1, [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 1, null, uuidBytes(UUID(1))]], null, null]
)
try {
decodeFact(payload, 2)
expect.unreachable('decode must throw')
} catch (error) {
const typed = error as UnknownLogRecordError
expect(typed).toBeInstanceOf(UnknownLogRecordError)
expect(typed.recordType).toBe(LOG_RECORD_TYPES.NOUN_TOMBSTONE)
expect(typed.recordVersion).toBe(1)
expect(typed.message).toMatch(/cipherFlag 1/)
expect(typed.message).toMatch(/encrypted records need a newer reader/)
}
})
it('a non-nil keyId refuses the same way, even with cipherFlag 0', () => {
const payload = encode(
[
1,
1,
[[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, uuidBytes(UUID(9)), uuidBytes(UUID(1))]],
null,
null
]
)
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
expect(() => decodeFact(payload, 2)).toThrow(/encrypted records need a newer reader/)
})
it('the encoder always writes the plaintext envelope: cipherFlag 0, keyId nil', () => {
const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })))
const raw = decode(payload) as unknown[]
const record = (raw[2] as unknown[][])[0]
expect(record[2]).toBe(LOG_RECORD_CIPHER_PLAINTEXT)
expect(record[3]).toBeNull()
expect(LOG_RECORD_CIPHER_PLAINTEXT).toBe(0)
})
it('an unknown segment format version has no decode path', () => {
const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })))
expect(() => decodeFact(payload, 3)).toThrow(/reads 1 and 2/)
@ -424,8 +469,8 @@ describe('fact-log format v2 — log.genesis width law', () => {
1,
1,
[
[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, uuidBytes(UUID(1))],
[LOG_RECORD_TYPES.LOG_GENESIS, 1, 64, uuidBytes(UUID(9)), 1]
[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))],
[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 64, uuidBytes(UUID(9)), 1]
],
null,
null
@ -434,7 +479,9 @@ describe('fact-log format v2 — log.genesis width law', () => {
})
it('an invalid genesis width on the wire is malformed, not a mismatch', () => {
const crafted = encode([1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 48, uuidBytes(UUID(9)), 1]], null, null])
const crafted = encode(
[1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 48, uuidBytes(UUID(9)), 1]], null, null]
)
expect(() => decodeFact(crafted, 2)).toThrow(/32 or 64/)
})
})
@ -504,7 +551,7 @@ describe('fact-log format v2 — vector legs (single-hop law)', () => {
})
expect(() => encodeFactV2(bad)).toThrow(/INLINE/)
const craftedRef = encode(
[1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, uuidBytes(UUID(7)), ['ref', 5]]], null, null]
[1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, 0, null, uuidBytes(UUID(7)), ['ref', 5]]], null, null]
)
expect(() => decodeFact(craftedRef, 2)).toThrow(/INLINE/)
})
@ -571,16 +618,30 @@ describe('fact-log format v2 — sector seals', () => {
timestamp: 1_700_000_000_123,
records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }]
})
const sealed = sealGroup([tomb], 64) // 51 bytes → gap 13 → overshoot → 77-byte pad
const sealed = sealGroup([tomb], 64) // 53 bytes → gap 11 → overshoot → 75-byte pad
expect(sealed.length).toBe(128)
expect(hex(sealed.subarray(tomb.length))).toBe(
// frame prefix + [0, 0, [[0, 1, bin8(42 zero bytes)]], nil, nil]
'450000009463044d95cf0000000000000000cf000000000000000091930001c42a' +
'0'.repeat(84) +
// frame prefix + [0, 0, [[0, 1, bin8(40 zero bytes)]], nil, nil]
'4300000088b4c8fa95cf0000000000000000cf000000000000000091930001c428' +
'0'.repeat(80) +
'c0c0'
)
})
it('encodePadFrame builds exact-size pads for streaming writers; refuses sub-minimum sizes', () => {
// Pads are envelope-exempt (skipped wholesale), so the smallest pad frame
// is byte-stable across the crypto-envelope change.
expect(minPadFrameBytes()).toBe(33)
for (const size of [minPadFrameBytes(), 64, 4096]) {
const pad = encodePadFrame(size)
expect(pad.length).toBe(size)
const { facts: decoded, validBytes } = decodeGroupV2(pad)
expect(decoded).toEqual([]) // invisible to readers
expect(validBytes).toBe(size)
}
expect(() => encodePadFrame(minPadFrameBytes() - 1)).toThrow(/at least/)
})
it('sealGroup refuses garbage: empty groups, malformed frames, bad seal sizes', () => {
expect(() => sealGroup([], 4096)).toThrow(/at least one frame/)
expect(() => sealGroup([new Uint8Array([1, 2, 3])], 4096)).toThrow(/not a well-formed frame/)
@ -620,10 +681,12 @@ describe('fact-log format v2 — torn-tail discipline', () => {
describe('fact-log format v2 — writer refusals (loud, never silent)', () => {
const tombstone = (g: number): CommitFactV2 => factOf(g, { type: 'noun.tombstone', id: UUID(g) })
it('refuses empty records, generation 0, and a second batch.meta', () => {
expect(() => encodeFactV2({ generation: 1, timestamp: 1, records: [] })).toThrow(
/at least one record/
)
it('accepts empty records (an all-deduped batch is a real generation); refuses generation 0 and a second batch.meta', () => {
// Contract change with the live cutover: v1 always encoded op-less
// commits (a batch whose relates dedupe away still mints a generation);
// v2 must not fork commit semantics — empty records round-trip.
const empty = decodeFact(framePayload(encodeFactV2({ generation: 1, timestamp: 1, records: [] })), 2)
expect(empty.records).toEqual([])
expect(() => encodeFactV2({ ...tombstone(1), generation: 0 })).toThrow(/positive integer/)
expect(() =>
encodeFactV2({