David Snelling
26c6025158
The cutover: new tail segments write format v2 (per-record [type, version, cipherFlag, keyId] envelope; noun/verb after-images carry dense ints MINTED AT APPEND from the id mapper — a rebuilt mapper reproduces assignments exactly; log.genesis opens every new log with the id-space width + a minted brain id; sync() seals to the header-declared sector boundary with reader-invisible pad frames). Existing v1 segments are never rewritten — per-segment decoder dispatch reads both formats and v2 facts map to the exact CommitFact shape all consumers already read. Cutover on a live v1 log: an empty v1 tail re-heads in place; a non-empty one is sealed by rotation, byte-identical. Records reserve the encryption fields (cipherFlag 0 / keyId nil are the only legal values; anything else refuses typed naming the needed newer reader) — crypto-ready with no future bump on the compat surface. Empty-records facts are legal (an all-deduped batch is a real generation — v1 semantics preserved; the refusal there tore a column-store flush mid-commit in the full suite, the consistency guard caught it loudly, and the root is fixed). Golden byte vectors pinned for the second (native) reader implementation. Pins: cutover 5/5 · codec 54 · kill-matrix stays 11/11.
808 lines
29 KiB
TypeScript
808 lines
29 KiB
TypeScript
/**
|
|
* @module tests/unit/db/factLogFormat
|
|
* @description Fact-log format v2 (record envelope + sector seals) pinned at
|
|
* the byte level: every record type round-trips field-exact (bigint ints,
|
|
* bin16 uuids, float-exact vectors), headers read v1 AND v2, unknown record
|
|
* types/versions refuse loudly with the typed error, the reserved crypto
|
|
* envelope (cipherFlag/keyId — plaintext-only this release) refuses anything
|
|
* nonzero/non-nil with the same typed error, genesis width mismatches
|
|
* refuse naming both widths, sealed groups align to the sector size with
|
|
* invisible pads, vector refs are writer-enforced single-hop, and torn tails
|
|
* truncate to the intact prefix at EVERY byte offset. This module is the
|
|
* reference implementation of a two-implementation contract — golden byte
|
|
* vectors here are frozen; a change that breaks them is a format change.
|
|
*/
|
|
import { describe, it, expect } from 'vitest'
|
|
import { encode, decode } from '@msgpack/msgpack'
|
|
import {
|
|
encodeFactV2,
|
|
decodeFact,
|
|
decodeGroupV2,
|
|
encodeSegmentHeaderV2,
|
|
parseSegmentHeader,
|
|
sealGroup,
|
|
framePayload,
|
|
encodePadFrame,
|
|
minPadFrameBytes,
|
|
UnknownLogRecordError,
|
|
GenesisWidthMismatchError,
|
|
LOG_RECORD_TYPES,
|
|
LOG_RECORD_VERSION,
|
|
LOG_RECORD_CIPHER_PLAINTEXT,
|
|
FACT_LOG_FORMAT_V1,
|
|
FACT_LOG_FORMAT_V2,
|
|
SEGMENT_HEADER_BYTES,
|
|
DEFAULT_SEAL_SIZE,
|
|
type CommitFactV2,
|
|
type LogRecord,
|
|
type VectorRef
|
|
} from '../../../src/db/factLogFormat.js'
|
|
|
|
const UUID = (n: number): string =>
|
|
`00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
|
|
const HASH_A = 'ab'.repeat(32)
|
|
const HASH_B = '0123456789abcdef'.repeat(4)
|
|
|
|
/** uuid string → bin16 (test-local mirror of the wire helper). */
|
|
const uuidBytes = (id: string): Uint8Array => {
|
|
const hex = id.replace(/-/g, '')
|
|
const bytes = new Uint8Array(16)
|
|
for (let i = 0; i < 16; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16)
|
|
return bytes
|
|
}
|
|
|
|
const hex = (bytes: Uint8Array): string => Buffer.from(bytes).toString('hex')
|
|
|
|
/** Encode → strip frame → decode; the standard round-trip. */
|
|
const roundTrip = (
|
|
fact: CommitFactV2,
|
|
encOpts?: Parameters<typeof encodeFactV2>[1],
|
|
decOpts?: { expectedIdSpaceWidth?: 32 | 64 }
|
|
): CommitFactV2 => decodeFact(framePayload(encodeFactV2(fact, encOpts)), 2, decOpts)
|
|
|
|
/** A single-record fact around `record`, canonical shape for strict equality. */
|
|
const factOf = (generation: number, record: LogRecord): CommitFactV2 => ({
|
|
generation,
|
|
timestamp: 1_700_000_000_000 + generation,
|
|
records: [record]
|
|
})
|
|
|
|
/**
|
|
* Build a fact frame of EXACTLY `totalBytes` (projection.note binary filler),
|
|
* for engineering precise seal-boundary scenarios.
|
|
*/
|
|
function frameOfExactly(totalBytes: number, generation: number): Uint8Array {
|
|
let fillerLength = Math.max(0, totalBytes - 60)
|
|
for (let i = 0; i < 12; i++) {
|
|
const frame = encodeFactV2({
|
|
generation,
|
|
timestamp: 1,
|
|
records: [{ type: 'projection.note', note: { fill: new Uint8Array(fillerLength) } }]
|
|
})
|
|
const diff = totalBytes - frame.length
|
|
if (diff === 0) return frame
|
|
fillerLength += diff
|
|
if (fillerLength < 0) throw new Error(`no frame of ${totalBytes} bytes is constructible`)
|
|
}
|
|
throw new Error('frame sizing did not converge')
|
|
}
|
|
|
|
describe('fact-log format v2 — record round-trips (field-exact)', () => {
|
|
it('noun.afterImage: bin16 uuid, u64-as-bigint beyond 2^53, metadata, inline vector', () => {
|
|
const fact = factOf(1, {
|
|
type: 'noun.afterImage',
|
|
id: UUID(1),
|
|
entityInt: (1n << 60n) + 3n, // provably beyond Number territory
|
|
metadata: {
|
|
noun: 'document',
|
|
title: 'doc 1',
|
|
nested: { tags: ['a', 'b'], score: 0.25 },
|
|
big: Number.MAX_SAFE_INTEGER,
|
|
negative: -42,
|
|
flag: true,
|
|
missing: null
|
|
},
|
|
vectorLeg: [0.1, -2.5, 3, 1e-7]
|
|
})
|
|
expect(roundTrip(fact)).toStrictEqual(fact)
|
|
})
|
|
|
|
it('noun.tombstone: body-less removal', () => {
|
|
const fact = factOf(2, { type: 'noun.tombstone', id: UUID(2) })
|
|
expect(roundTrip(fact)).toStrictEqual(fact)
|
|
})
|
|
|
|
it('verb.afterImage: both endpoints, three u64 handles, verb name', () => {
|
|
const fact = factOf(3, {
|
|
type: 'verb.afterImage',
|
|
id: UUID(3),
|
|
verbInt: 18_446_744_073_709_551_615n, // u64 max
|
|
metadata: { verb: 'contains', weight: 0.5 },
|
|
vectorLeg: null,
|
|
verb: 'contains',
|
|
sourceId: UUID(31),
|
|
sourceInt: 7n,
|
|
targetId: UUID(32),
|
|
targetInt: (1n << 53n) + 1n
|
|
})
|
|
expect(roundTrip(fact)).toStrictEqual(fact)
|
|
})
|
|
|
|
it('verb.tombstone: body-less removal', () => {
|
|
const fact = factOf(4, { type: 'verb.tombstone', id: UUID(4) })
|
|
expect(roundTrip(fact)).toStrictEqual(fact)
|
|
})
|
|
|
|
it('batch.meta: one metadata map per fact', () => {
|
|
const fact = factOf(5, { type: 'batch.meta', meta: { source: 'import', count: 12 } })
|
|
expect(roundTrip(fact)).toStrictEqual(fact)
|
|
})
|
|
|
|
it('embed.pending: id + enqueue time', () => {
|
|
const fact = factOf(6, { type: 'embed.pending', id: UUID(6), enqueuedAt: 1_700_000_000_777 })
|
|
expect(roundTrip(fact)).toStrictEqual(fact)
|
|
})
|
|
|
|
it('embed.landed: inline vector, float-exact', () => {
|
|
const fact = factOf(7, {
|
|
type: 'embed.landed',
|
|
id: UUID(7),
|
|
vector: [0.30000000000000004, -1.5, 2 ** 31 + 0.5]
|
|
})
|
|
expect(roundTrip(fact)).toStrictEqual(fact)
|
|
})
|
|
|
|
it('blob.manifest: bin32 hash, size, mimeType, both refOps', () => {
|
|
const add = factOf(8, {
|
|
type: 'blob.manifest',
|
|
hash: HASH_A,
|
|
size: 1_048_576,
|
|
mimeType: 'image/png',
|
|
refOp: 'add'
|
|
})
|
|
expect(roundTrip(add)).toStrictEqual(add)
|
|
const release = factOf(9, {
|
|
type: 'blob.manifest',
|
|
hash: HASH_B,
|
|
size: 0,
|
|
mimeType: 'application/octet-stream',
|
|
refOp: 'release'
|
|
})
|
|
expect(roundTrip(release)).toStrictEqual(release)
|
|
})
|
|
|
|
it('projection.note: opaque map rides untouched', () => {
|
|
const fact = factOf(10, {
|
|
type: 'projection.note',
|
|
note: { consumer: 'reserved', payload: { depth: [1, 2, 3] } }
|
|
})
|
|
expect(roundTrip(fact)).toStrictEqual(fact)
|
|
})
|
|
|
|
it('bootstrap.baseline: kind flag, metadata, vector leg — both kinds', () => {
|
|
const noun = factOf(11, {
|
|
type: 'bootstrap.baseline',
|
|
id: UUID(11),
|
|
kind: 'noun',
|
|
metadata: { noun: 'person' },
|
|
vectorLeg: [1, 2, 3]
|
|
})
|
|
expect(roundTrip(noun)).toStrictEqual(noun)
|
|
const verb = factOf(12, {
|
|
type: 'bootstrap.baseline',
|
|
id: UUID(12),
|
|
kind: 'verb',
|
|
metadata: null,
|
|
vectorLeg: null
|
|
})
|
|
expect(roundTrip(verb)).toStrictEqual(verb)
|
|
})
|
|
|
|
it('log.genesis: width, brainId, createdAt — both widths', () => {
|
|
for (const idSpaceWidth of [32, 64] as const) {
|
|
const fact = factOf(1, {
|
|
type: 'log.genesis',
|
|
idSpaceWidth,
|
|
brainId: UUID(999),
|
|
createdAt: 1_700_000_000_000
|
|
})
|
|
expect(roundTrip(fact, undefined, { expectedIdSpaceWidth: idSpaceWidth })).toStrictEqual(fact)
|
|
}
|
|
})
|
|
|
|
it('a combined fact: genesis-first, all record types, fact meta, duplicate blobHashes', () => {
|
|
const fact: CommitFactV2 = {
|
|
generation: 1,
|
|
timestamp: 1_700_000_000_001,
|
|
records: [
|
|
{ type: 'log.genesis', idSpaceWidth: 64, brainId: UUID(999), createdAt: 1_699_999_999_999 },
|
|
{ type: 'noun.afterImage', id: UUID(1), entityInt: 1n, metadata: { a: 1 }, vectorLeg: [0.5] },
|
|
{ type: 'noun.tombstone', id: UUID(2) },
|
|
{
|
|
type: 'verb.afterImage',
|
|
id: UUID(3),
|
|
verbInt: 3n,
|
|
metadata: null,
|
|
vectorLeg: null,
|
|
verb: 'relatedTo',
|
|
sourceId: UUID(31),
|
|
sourceInt: 1n,
|
|
targetId: UUID(32),
|
|
targetInt: 2n
|
|
},
|
|
{ type: 'verb.tombstone', id: UUID(4) },
|
|
{ type: 'batch.meta', meta: { origin: 'unit' } },
|
|
{ type: 'embed.pending', id: UUID(6), enqueuedAt: 5 },
|
|
{ type: 'embed.landed', id: UUID(7), vector: [0.1] },
|
|
{ type: 'blob.manifest', hash: HASH_A, size: 9, mimeType: 'text/plain', refOp: 'add' },
|
|
{ type: 'projection.note', note: {} },
|
|
{ type: 'bootstrap.baseline', id: UUID(11), kind: 'noun', metadata: null, vectorLeg: null }
|
|
],
|
|
meta: { source: 'unit' },
|
|
blobHashes: [HASH_A, HASH_A] // multiset — duplicates preserved
|
|
}
|
|
expect(roundTrip(fact, undefined, { expectedIdSpaceWidth: 64 })).toStrictEqual(fact)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — golden byte vectors (frozen contract)', () => {
|
|
it('v2 segment header bytes are pinned', () => {
|
|
expect(hex(encodeSegmentHeaderV2(7, 4096))).toBe(
|
|
'4246414354530000020000000700000000000000001000000000000000000000'
|
|
)
|
|
})
|
|
|
|
it('a noun.tombstone frame is pinned byte-for-byte', () => {
|
|
const frame = encodeFactV2({
|
|
generation: 3,
|
|
timestamp: 1_700_000_000_123,
|
|
records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }]
|
|
})
|
|
expect(hex(frame)).toBe(
|
|
'2d00000048e4d43695cf0000000000000003cf0000018bcfe5687b9195020100c0' +
|
|
'c41000000000000040008000000000000042c0c0'
|
|
)
|
|
})
|
|
|
|
it('u64 registry fields ride as fixed 8-byte msgpack uint64 (0xcf)', () => {
|
|
const payload = framePayload(
|
|
encodeFactV2(factOf(1, { type: 'embed.pending', id: UUID(1), enqueuedAt: 2 }))
|
|
)
|
|
// positions 0 and 1 (generation, timestamp) and enqueuedAt are all 0xcf
|
|
expect(payload[1]).toBe(0xcf)
|
|
expect(payload[10]).toBe(0xcf)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — segment headers (v1 AND v2)', () => {
|
|
const v1Header = (): Uint8Array => {
|
|
const header = new Uint8Array(SEGMENT_HEADER_BYTES)
|
|
header.set(new Uint8Array([0x42, 0x46, 0x41, 0x43, 0x54, 0x53, 0x00, 0x00]), 0)
|
|
const view = new DataView(header.buffer)
|
|
view.setUint32(8, FACT_LOG_FORMAT_V1, true)
|
|
view.setBigUint64(12, 42n, true)
|
|
return header
|
|
}
|
|
|
|
it('a v2 header round-trips with its sealSize', () => {
|
|
const header = encodeSegmentHeaderV2(123_456, 512)
|
|
expect(header.length).toBe(SEGMENT_HEADER_BYTES)
|
|
expect(parseSegmentHeader(header)).toStrictEqual({
|
|
formatVersion: FACT_LOG_FORMAT_V2,
|
|
firstGeneration: 123_456,
|
|
sealSize: 512
|
|
})
|
|
// default sealSize
|
|
expect(parseSegmentHeader(encodeSegmentHeaderV2(1)).sealSize).toBe(DEFAULT_SEAL_SIZE)
|
|
})
|
|
|
|
it('a v1 header parses: version 1, sealSize absent (undefined)', () => {
|
|
const parsed = parseSegmentHeader(v1Header())
|
|
expect(parsed).toStrictEqual({ formatVersion: FACT_LOG_FORMAT_V1, firstGeneration: 42 })
|
|
expect(parsed.sealSize).toBeUndefined()
|
|
})
|
|
|
|
it('corrupted magic throws', () => {
|
|
const header = encodeSegmentHeaderV2(1)
|
|
header[0] = 0x58
|
|
expect(() => parseSegmentHeader(header)).toThrow(/bad magic/)
|
|
})
|
|
|
|
it('non-zero reserved bytes throw — v1 (offset 20+) and v2 (offset 22+)', () => {
|
|
const v1 = v1Header()
|
|
v1[21] = 1
|
|
expect(() => parseSegmentHeader(v1)).toThrow(/non-zero reserved/)
|
|
|
|
const v2 = encodeSegmentHeaderV2(1, 4096)
|
|
v2[25] = 1
|
|
expect(() => parseSegmentHeader(v2)).toThrow(/non-zero reserved/)
|
|
})
|
|
|
|
it('the v2 sealSize bytes are NOT reserved bytes in v2 (but ARE in v1)', () => {
|
|
// sealSize 512 puts a non-zero byte at offset 21 — legal in v2 only.
|
|
const v2 = encodeSegmentHeaderV2(1, 512)
|
|
expect(parseSegmentHeader(v2).sealSize).toBe(512)
|
|
const v1 = v1Header()
|
|
v1[20] = 0x00
|
|
v1[21] = 0x02 // same bytes a v2 sealSize=512 would carry
|
|
expect(() => parseSegmentHeader(v1)).toThrow(/non-zero reserved/)
|
|
})
|
|
|
|
it('an unknown header version and a short buffer throw', () => {
|
|
const header = encodeSegmentHeaderV2(1)
|
|
new DataView(header.buffer).setUint32(8, 3, true)
|
|
expect(() => parseSegmentHeader(header)).toThrow(/formatVersion 3/)
|
|
expect(() => parseSegmentHeader(header.subarray(0, 31))).toThrow(/32 bytes/)
|
|
})
|
|
|
|
it('header writer refuses out-of-range inputs', () => {
|
|
expect(() => encodeSegmentHeaderV2(-1)).toThrow(/non-negative/)
|
|
expect(() => encodeSegmentHeaderV2(1, 32)).toThrow(/sealSize/)
|
|
expect(() => encodeSegmentHeaderV2(1, 65_536)).toThrow(/sealSize/)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — decoder law (typed refusals, never skip)', () => {
|
|
it('unknown record type 12 throws UnknownLogRecordError naming type 12', () => {
|
|
const payload = encode([1, 1, [[12, 1]], null, null])
|
|
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
|
|
try {
|
|
decodeFact(payload, 2)
|
|
expect.unreachable('decode must throw')
|
|
} catch (error) {
|
|
const typed = error as UnknownLogRecordError
|
|
expect(typed).toBeInstanceOf(UnknownLogRecordError)
|
|
expect(typed.recordType).toBe(12)
|
|
expect(typed.recordVersion).toBe(1)
|
|
expect(typed.message).toMatch(/type 12/)
|
|
expect(typed.message).toMatch(/newer reader/)
|
|
}
|
|
})
|
|
|
|
it('recordVersion 2 on a known type throws the same class naming the version', () => {
|
|
const payload = encode([1, 1, [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 2, new Uint8Array(16)]], null, null])
|
|
try {
|
|
decodeFact(payload, 2)
|
|
expect.unreachable('decode must throw')
|
|
} catch (error) {
|
|
const typed = error as UnknownLogRecordError
|
|
expect(typed).toBeInstanceOf(UnknownLogRecordError)
|
|
expect(typed.recordType).toBe(LOG_RECORD_TYPES.NOUN_TOMBSTONE)
|
|
expect(typed.recordVersion).toBe(2)
|
|
expect(typed.message).toMatch(/version 2/)
|
|
expect(typed.message).toMatch(/newer reader/)
|
|
}
|
|
})
|
|
|
|
it('a fact mixing known and unknown records still refuses (no partial reads)', () => {
|
|
const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))]
|
|
const payload = encode([1, 1, [known, [200, 1]], null, null])
|
|
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
|
|
})
|
|
|
|
it('a nonzero cipherFlag refuses with the typed error — encrypted records need a newer reader', () => {
|
|
const payload = encode(
|
|
[1, 1, [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 1, null, uuidBytes(UUID(1))]], null, null]
|
|
)
|
|
try {
|
|
decodeFact(payload, 2)
|
|
expect.unreachable('decode must throw')
|
|
} catch (error) {
|
|
const typed = error as UnknownLogRecordError
|
|
expect(typed).toBeInstanceOf(UnknownLogRecordError)
|
|
expect(typed.recordType).toBe(LOG_RECORD_TYPES.NOUN_TOMBSTONE)
|
|
expect(typed.recordVersion).toBe(1)
|
|
expect(typed.message).toMatch(/cipherFlag 1/)
|
|
expect(typed.message).toMatch(/encrypted records need a newer reader/)
|
|
}
|
|
})
|
|
|
|
it('a non-nil keyId refuses the same way, even with cipherFlag 0', () => {
|
|
const payload = encode(
|
|
[
|
|
1,
|
|
1,
|
|
[[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, uuidBytes(UUID(9)), uuidBytes(UUID(1))]],
|
|
null,
|
|
null
|
|
]
|
|
)
|
|
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
|
|
expect(() => decodeFact(payload, 2)).toThrow(/encrypted records need a newer reader/)
|
|
})
|
|
|
|
it('the encoder always writes the plaintext envelope: cipherFlag 0, keyId nil', () => {
|
|
const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })))
|
|
const raw = decode(payload) as unknown[]
|
|
const record = (raw[2] as unknown[][])[0]
|
|
expect(record[2]).toBe(LOG_RECORD_CIPHER_PLAINTEXT)
|
|
expect(record[3]).toBeNull()
|
|
expect(LOG_RECORD_CIPHER_PLAINTEXT).toBe(0)
|
|
})
|
|
|
|
it('an unknown segment format version has no decode path', () => {
|
|
const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })))
|
|
expect(() => decodeFact(payload, 3)).toThrow(/reads 1 and 2/)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — log.genesis width law', () => {
|
|
const genesisFact = (width: 32 | 64): CommitFactV2 =>
|
|
factOf(1, { type: 'log.genesis', idSpaceWidth: width, brainId: UUID(9), createdAt: 1 })
|
|
|
|
it('expectedWidth 32 vs a 64-width genesis refuses, naming both widths', () => {
|
|
const payload = framePayload(encodeFactV2(genesisFact(64)))
|
|
expect(() => decodeFact(payload, 2, { expectedIdSpaceWidth: 32 })).toThrow(
|
|
GenesisWidthMismatchError
|
|
)
|
|
try {
|
|
decodeFact(payload, 2, { expectedIdSpaceWidth: 32 })
|
|
expect.unreachable('decode must throw')
|
|
} catch (error) {
|
|
const typed = error as GenesisWidthMismatchError
|
|
expect(typed.expectedWidth).toBe(32)
|
|
expect(typed.actualWidth).toBe(64)
|
|
expect(typed.message).toMatch(/32-bit/)
|
|
expect(typed.message).toMatch(/64-bit/)
|
|
}
|
|
})
|
|
|
|
it('a matching width (and no expectation at all) decodes cleanly', () => {
|
|
const payload = framePayload(encodeFactV2(genesisFact(64)))
|
|
expect(decodeFact(payload, 2, { expectedIdSpaceWidth: 64 }).records[0]).toMatchObject({
|
|
idSpaceWidth: 64
|
|
})
|
|
expect(decodeFact(payload, 2).records[0]).toMatchObject({ idSpaceWidth: 64 })
|
|
})
|
|
|
|
it('genesis anywhere but record 0 refuses — encode AND decode', () => {
|
|
const late: CommitFactV2 = {
|
|
generation: 1,
|
|
timestamp: 1,
|
|
records: [
|
|
{ type: 'noun.tombstone', id: UUID(1) },
|
|
{ type: 'log.genesis', idSpaceWidth: 64, brainId: UUID(9), createdAt: 1 }
|
|
]
|
|
}
|
|
expect(() => encodeFactV2(late)).toThrow(/first record/)
|
|
const crafted = encode([
|
|
1,
|
|
1,
|
|
[
|
|
[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))],
|
|
[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 64, uuidBytes(UUID(9)), 1]
|
|
],
|
|
null,
|
|
null
|
|
])
|
|
expect(() => decodeFact(crafted, 2)).toThrow(/first record/)
|
|
})
|
|
|
|
it('an invalid genesis width on the wire is malformed, not a mismatch', () => {
|
|
const crafted = encode(
|
|
[1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 48, uuidBytes(UUID(9)), 1]], null, null]
|
|
)
|
|
expect(() => decodeFact(crafted, 2)).toThrow(/32 or 64/)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — vector legs (single-hop law)', () => {
|
|
it('inline vectors round-trip float-exact', () => {
|
|
const vector = [0.1 + 0.2, -0.0000001, 3.141592653589793, 2 ** 40 + 0.25]
|
|
const fact = factOf(1, {
|
|
type: 'noun.afterImage',
|
|
id: UUID(1),
|
|
entityInt: 1n,
|
|
metadata: null,
|
|
vectorLeg: vector
|
|
})
|
|
const decoded = roundTrip(fact)
|
|
expect((decoded.records[0] as { vectorLeg: number[] }).vectorLeg).toStrictEqual(vector)
|
|
})
|
|
|
|
it('a ref round-trips when the validator vouches for the target generation', () => {
|
|
const fact = factOf(6, {
|
|
type: 'noun.afterImage',
|
|
id: UUID(1),
|
|
entityInt: 1n,
|
|
metadata: null,
|
|
vectorLeg: { sameAsGeneration: 5 }
|
|
})
|
|
const viaSet = roundTrip(fact, { inlineVectorGenerations: new Set([5]) })
|
|
expect((viaSet.records[0] as { vectorLeg: VectorRef }).vectorLeg).toStrictEqual({
|
|
sameAsGeneration: 5
|
|
})
|
|
const viaCallback = roundTrip(fact, { inlineVectorGenerations: (g) => g === 5 })
|
|
expect(viaCallback).toStrictEqual(fact)
|
|
})
|
|
|
|
it('the encoder REFUSES a ref the validator rejects', () => {
|
|
const fact = factOf(6, {
|
|
type: 'noun.afterImage',
|
|
id: UUID(1),
|
|
entityInt: 1n,
|
|
metadata: null,
|
|
vectorLeg: { sameAsGeneration: 5 }
|
|
})
|
|
expect(() => encodeFactV2(fact, { inlineVectorGenerations: new Set([4]) })).toThrow(
|
|
/single-hop/
|
|
)
|
|
expect(() => encodeFactV2(fact, { inlineVectorGenerations: () => false })).toThrow(
|
|
/generation 5/
|
|
)
|
|
})
|
|
|
|
it('the encoder REFUSES a ref when no validator was provided at all', () => {
|
|
const fact = factOf(6, {
|
|
type: 'noun.afterImage',
|
|
id: UUID(1),
|
|
entityInt: 1n,
|
|
metadata: null,
|
|
vectorLeg: { sameAsGeneration: 5 }
|
|
})
|
|
expect(() => encodeFactV2(fact)).toThrow(/unverifiable ref/)
|
|
})
|
|
|
|
it('embed.landed is inline-only: encode refuses non-arrays, decode refuses wire refs', () => {
|
|
const bad = factOf(7, {
|
|
type: 'embed.landed',
|
|
id: UUID(7),
|
|
vector: null as unknown as number[]
|
|
})
|
|
expect(() => encodeFactV2(bad)).toThrow(/INLINE/)
|
|
const craftedRef = encode(
|
|
[1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, 0, null, uuidBytes(UUID(7)), ['ref', 5]]], null, null]
|
|
)
|
|
expect(() => decodeFact(craftedRef, 2)).toThrow(/INLINE/)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — sector seals', () => {
|
|
const facts = [1, 2, 3].map((g) =>
|
|
factOf(g, {
|
|
type: 'noun.afterImage',
|
|
id: UUID(g),
|
|
entityInt: BigInt(g),
|
|
metadata: { title: `doc ${g}` },
|
|
vectorLeg: [g + 0.5]
|
|
})
|
|
)
|
|
const frames = facts.map((f) => encodeFactV2(f))
|
|
|
|
it('sealGroup output is sector-aligned and decodes to exactly the input facts', () => {
|
|
const sealed = sealGroup(frames, 4096)
|
|
expect(sealed.length % 4096).toBe(0)
|
|
const { facts: decoded, validBytes } = decodeGroupV2(sealed)
|
|
expect(decoded).toStrictEqual(facts) // pads invisible
|
|
expect(validBytes).toBe(sealed.length)
|
|
})
|
|
|
|
it('an already-aligned group gets NO pad (byte-identical passthrough)', () => {
|
|
const exact = frameOfExactly(4096, 1)
|
|
const sealed = sealGroup([exact], 4096)
|
|
expect(sealed.length).toBe(4096)
|
|
expect(Buffer.compare(Buffer.from(sealed), Buffer.from(exact))).toBe(0)
|
|
expect(decodeGroupV2(sealed).facts).toHaveLength(1)
|
|
})
|
|
|
|
it('a normal gap gets ONE exact-fit pad frame', () => {
|
|
const sealed = sealGroup([frameOfExactly(2000, 1), frameOfExactly(1996, 2)], 4096) // gap 100
|
|
expect(sealed.length).toBe(4096)
|
|
expect(decodeGroupV2(sealed).facts.map((f) => f.generation)).toEqual([1, 2])
|
|
})
|
|
|
|
it('a gap too small for any frame (the <12-byte remainder and friends) pads through one extra sector', () => {
|
|
for (const gap of [1, 8, 11, 16, 32]) {
|
|
const sealed = sealGroup([frameOfExactly(4096 - gap, 1)], 4096)
|
|
expect(sealed.length % 4096).toBe(0)
|
|
expect(sealed.length).toBe(8192) // gap + one full sector, still aligned
|
|
const { facts: decoded, validBytes } = decodeGroupV2(sealed)
|
|
expect(decoded.map((f) => f.generation)).toEqual([1])
|
|
expect(validBytes).toBe(8192)
|
|
}
|
|
// the smallest constructible pad frame fits exactly — no overshoot at 33
|
|
const sealed33 = sealGroup([frameOfExactly(4096 - 33, 1)], 4096)
|
|
expect(sealed33.length).toBe(4096)
|
|
expect(decodeGroupV2(sealed33).facts.map((f) => f.generation)).toEqual([1])
|
|
})
|
|
|
|
it('seals honor a custom sealSize (device-probed sizes are the caller business)', () => {
|
|
const sealed = sealGroup(frames, 512)
|
|
expect(sealed.length % 512).toBe(0)
|
|
expect(decodeGroupV2(sealed).facts).toStrictEqual(facts)
|
|
})
|
|
|
|
it('pad frame bytes are pinned (golden vector, sealSize 64)', () => {
|
|
const tomb = encodeFactV2({
|
|
generation: 3,
|
|
timestamp: 1_700_000_000_123,
|
|
records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }]
|
|
})
|
|
const sealed = sealGroup([tomb], 64) // 53 bytes → gap 11 → overshoot → 75-byte pad
|
|
expect(sealed.length).toBe(128)
|
|
expect(hex(sealed.subarray(tomb.length))).toBe(
|
|
// frame prefix + [0, 0, [[0, 1, bin8(40 zero bytes)]], nil, nil]
|
|
'4300000088b4c8fa95cf0000000000000000cf000000000000000091930001c428' +
|
|
'0'.repeat(80) +
|
|
'c0c0'
|
|
)
|
|
})
|
|
|
|
it('encodePadFrame builds exact-size pads for streaming writers; refuses sub-minimum sizes', () => {
|
|
// Pads are envelope-exempt (skipped wholesale), so the smallest pad frame
|
|
// is byte-stable across the crypto-envelope change.
|
|
expect(minPadFrameBytes()).toBe(33)
|
|
for (const size of [minPadFrameBytes(), 64, 4096]) {
|
|
const pad = encodePadFrame(size)
|
|
expect(pad.length).toBe(size)
|
|
const { facts: decoded, validBytes } = decodeGroupV2(pad)
|
|
expect(decoded).toEqual([]) // invisible to readers
|
|
expect(validBytes).toBe(size)
|
|
}
|
|
expect(() => encodePadFrame(minPadFrameBytes() - 1)).toThrow(/at least/)
|
|
})
|
|
|
|
it('sealGroup refuses garbage: empty groups, malformed frames, bad seal sizes', () => {
|
|
expect(() => sealGroup([], 4096)).toThrow(/at least one frame/)
|
|
expect(() => sealGroup([new Uint8Array([1, 2, 3])], 4096)).toThrow(/not a well-formed frame/)
|
|
const corrupted = encodeFactV2(facts[0])
|
|
corrupted[corrupted.length - 1] ^= 0xff
|
|
expect(() => sealGroup([corrupted], 4096)).toThrow(/not a well-formed frame/)
|
|
expect(() => sealGroup(frames, 32)).toThrow(/sealSize/)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — torn-tail discipline', () => {
|
|
it('truncating a sealed group at EVERY byte offset of the tail yields the intact prefix, never an uncontrolled throw', () => {
|
|
const frames = [frameOfExactly(600, 1), frameOfExactly(700, 2), frameOfExactly(800, 3)]
|
|
const sealed = sealGroup(frames, 4096)
|
|
expect(sealed.length).toBe(4096)
|
|
const f3End = 600 + 700 + 800
|
|
|
|
for (let cut = 600 + 700; cut < sealed.length; cut++) {
|
|
const { facts: decoded, validBytes } = decodeGroupV2(sealed.subarray(0, cut))
|
|
const expected = cut < f3End ? [1, 2] : [1, 2, 3]
|
|
expect(decoded.map((f) => f.generation)).toEqual(expected)
|
|
expect(validBytes).toBe(cut < f3End ? 600 + 700 : f3End)
|
|
}
|
|
})
|
|
|
|
it('a flipped payload byte (not just truncation) also terminates the walk at the damage', () => {
|
|
const frames = [frameOfExactly(600, 1), frameOfExactly(700, 2)]
|
|
const sealed = sealGroup(frames, 4096)
|
|
const damaged = sealed.slice()
|
|
damaged[600 + 100] ^= 0xff // inside frame 2's payload
|
|
const { facts: decoded, validBytes } = decodeGroupV2(damaged)
|
|
expect(decoded.map((f) => f.generation)).toEqual([1])
|
|
expect(validBytes).toBe(600)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — writer refusals (loud, never silent)', () => {
|
|
const tombstone = (g: number): CommitFactV2 => factOf(g, { type: 'noun.tombstone', id: UUID(g) })
|
|
|
|
it('accepts empty records (an all-deduped batch is a real generation); refuses generation 0 and a second batch.meta', () => {
|
|
// Contract change with the live cutover: v1 always encoded op-less
|
|
// commits (a batch whose relates dedupe away still mints a generation);
|
|
// v2 must not fork commit semantics — empty records round-trip.
|
|
const empty = decodeFact(framePayload(encodeFactV2({ generation: 1, timestamp: 1, records: [] })), 2)
|
|
expect(empty.records).toEqual([])
|
|
expect(() => encodeFactV2({ ...tombstone(1), generation: 0 })).toThrow(/positive integer/)
|
|
expect(() =>
|
|
encodeFactV2({
|
|
generation: 1,
|
|
timestamp: 1,
|
|
records: [
|
|
{ type: 'batch.meta', meta: { a: 1 } },
|
|
{ type: 'batch.meta', meta: { b: 2 } }
|
|
]
|
|
})
|
|
).toThrow(/at most one batch.meta/)
|
|
})
|
|
|
|
it('refuses pad records — filler belongs to sealGroup, not to writers', () => {
|
|
const fact = {
|
|
generation: 1,
|
|
timestamp: 1,
|
|
records: [{ type: 'pad' } as unknown as LogRecord]
|
|
}
|
|
expect(() => encodeFactV2(fact)).toThrow(/cannot encode record type pad/)
|
|
})
|
|
|
|
it('refuses malformed field values: non-uuid ids, bad hashes, out-of-range u64s', () => {
|
|
expect(() =>
|
|
encodeFactV2(factOf(1, { type: 'noun.tombstone', id: 'not-a-uuid' }))
|
|
).toThrow(/not a uuid/)
|
|
expect(() =>
|
|
encodeFactV2(
|
|
factOf(1, { type: 'blob.manifest', hash: 'abc', size: 1, mimeType: 'x', refOp: 'add' })
|
|
)
|
|
).toThrow(/64 hex chars/)
|
|
expect(() =>
|
|
encodeFactV2(
|
|
factOf(1, {
|
|
type: 'noun.afterImage',
|
|
id: UUID(1),
|
|
entityInt: -1n,
|
|
metadata: null,
|
|
vectorLeg: null
|
|
})
|
|
)
|
|
).toThrow(/u64 range/)
|
|
expect(() =>
|
|
encodeFactV2(
|
|
factOf(1, {
|
|
type: 'noun.afterImage',
|
|
id: UUID(1),
|
|
entityInt: 1n << 64n,
|
|
metadata: null,
|
|
vectorLeg: null
|
|
})
|
|
)
|
|
).toThrow(/u64 range/)
|
|
})
|
|
})
|
|
|
|
describe('fact-log format — the v1 decode path stays readable forever', () => {
|
|
it('decodeFact(payload, 1) reads the v1 ops shape (positional, bin16, tombstones)', () => {
|
|
// Crafted exactly as the v1 writer frames facts: default msgpack, ops at
|
|
// position 2 as [kind u8, id bin16, [metadata, vector] | nil].
|
|
const payload = encode([
|
|
4,
|
|
1_700_000_000_004,
|
|
[
|
|
[0, uuidBytes(UUID(41)), [{ noun: 'document', title: 'doc 41' }, { v: [1, 2] }]],
|
|
[1, uuidBytes(UUID(42)), null] // verb tombstone
|
|
],
|
|
{ source: 'v1' },
|
|
['abc123']
|
|
])
|
|
const fact = decodeFact(payload, 1)
|
|
expect(fact).toStrictEqual({
|
|
generation: 4,
|
|
timestamp: 1_700_000_000_004,
|
|
ops: [
|
|
{
|
|
kind: 'noun',
|
|
id: UUID(41),
|
|
record: { metadata: { noun: 'document', title: 'doc 41' }, vector: { v: [1, 2] } }
|
|
},
|
|
{ kind: 'verb', id: UUID(42), record: null }
|
|
],
|
|
meta: { source: 'v1' },
|
|
blobHashes: ['abc123']
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('fact-log format v2 — frame envelope helper', () => {
|
|
it('framePayload verifies exact length and crc32c', () => {
|
|
const frame = encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) }))
|
|
expect(() => framePayload(frame)).not.toThrow()
|
|
|
|
const shortFrame = frame.subarray(0, frame.length - 1)
|
|
expect(() => framePayload(shortFrame)).toThrow(/declares/)
|
|
|
|
const corrupted = frame.slice()
|
|
corrupted[corrupted.length - 1] ^= 0xff
|
|
expect(() => framePayload(corrupted)).toThrow(/crc32c/)
|
|
})
|
|
|
|
it('the record-type registry and version constants are the frozen wire codes', () => {
|
|
expect(LOG_RECORD_TYPES).toStrictEqual({
|
|
PAD: 0,
|
|
NOUN_AFTER_IMAGE: 1,
|
|
NOUN_TOMBSTONE: 2,
|
|
VERB_AFTER_IMAGE: 3,
|
|
VERB_TOMBSTONE: 4,
|
|
BATCH_META: 5,
|
|
EMBED_PENDING: 6,
|
|
EMBED_LANDED: 7,
|
|
BLOB_MANIFEST: 8,
|
|
PROJECTION_NOTE: 9,
|
|
BOOTSTRAP_BASELINE: 10,
|
|
LOG_GENESIS: 11
|
|
})
|
|
expect(LOG_RECORD_VERSION).toBe(1)
|
|
})
|
|
})
|