feat(log): v2 is the LIVE write format — envelope records with minted ints, genesis, sector seals; v1 readable forever
The cutover: new tail segments write format v2 (per-record [type, version, cipherFlag, keyId] envelope; noun/verb after-images carry dense ints MINTED AT APPEND from the id mapper — a rebuilt mapper reproduces assignments exactly; log.genesis opens every new log with the id-space width + a minted brain id; sync() seals to the header-declared sector boundary with reader-invisible pad frames). Existing v1 segments are never rewritten — per-segment decoder dispatch reads both formats and v2 facts map to the exact CommitFact shape all consumers already read. Cutover on a live v1 log: an empty v1 tail re-heads in place; a non-empty one is sealed by rotation, byte-identical. Records reserve the encryption fields (cipherFlag 0 / keyId nil are the only legal values; anything else refuses typed naming the needed newer reader) — crypto-ready with no future bump on the compat surface. Empty-records facts are legal (an all-deduped batch is a real generation — v1 semantics preserved; the refusal there tore a column-store flush mid-commit in the full suite, the consistency guard caught it loudly, and the root is fixed). Golden byte vectors pinned for the second (native) reader implementation. Pins: cutover 5/5 · codec 54 · kill-matrix stays 11/11.
This commit is contained in:
parent
73eb88d481
commit
26c6025158
6 changed files with 1372 additions and 135 deletions
|
|
@ -41,10 +41,57 @@
|
|||
* terminal-readable) is the single source of truth for the segment SET;
|
||||
* rotation flips it atomically (write-new → fsync → rename) BEFORE the new
|
||||
* tail's first byte exists, so no segment file is ever unaccounted for.
|
||||
*
|
||||
* ## Mixed-version logs (the v2 live-write cutover)
|
||||
*
|
||||
* The segment header's `formatVersion` selects the decoder PER SEGMENT:
|
||||
* v1 segments (ops-shaped facts, the format above) stay readable forever and
|
||||
* are NEVER rewritten; a NEW tail segment writes the v2 format
|
||||
* (`src/db/factLogFormat.ts` — record envelope, minted dense ints, genesis,
|
||||
* sector seals) whenever the int minter is installed ({@link FactLog.setIntMinter} —
|
||||
* the brain wires it from the metadata index's id mapper right after init).
|
||||
* A bare `FactLog` with no minter keeps writing v1 (there is no authority
|
||||
* that could reproduce int assignments, and 0 is never written). Cutover
|
||||
* mechanics on an existing v1 log: an EMPTY v1 tail is re-headed to v2 in
|
||||
* place; a non-empty v1 tail is sealed by an immediate rotation and the new
|
||||
* tail is v2. Decoded v2 facts map back to the SAME {@link CommitFact} shape
|
||||
* v1 consumers read (noun/verb ops with `{metadata, vector} | null` records) —
|
||||
* the vector wrapper object is reconstructed from the record's metadata leg
|
||||
* through the reserved-field hydration law (see `commitFactFromV2`).
|
||||
*
|
||||
* V2 tails additionally: write the `log.genesis` record (id-space width 64 +
|
||||
* the brain id, minted once into the manifest's additive `brainId` field) as
|
||||
* the first record of the FIRST fact of a brand-new log, and seal every
|
||||
* `sync()` to the header-declared sector size with pad frames that are
|
||||
* invisible to readers (torn-page defense at group-commit boundaries).
|
||||
*/
|
||||
import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack'
|
||||
import { crc32c } from '../utils/crc32c.js'
|
||||
import { prodLog } from '../utils/logger.js'
|
||||
import {
|
||||
FACT_LOG_FORMAT_V1,
|
||||
FACT_LOG_FORMAT_V2,
|
||||
DEFAULT_SEAL_SIZE,
|
||||
parseSegmentHeader,
|
||||
encodeSegmentHeaderV2,
|
||||
encodeFactV2,
|
||||
decodeFact as decodeFormatFact,
|
||||
decodeGroupV2,
|
||||
encodePadFrame,
|
||||
minPadFrameBytes,
|
||||
type CommitFactV2,
|
||||
type LogRecord,
|
||||
type EmbedPendingRecord,
|
||||
type EmbedLandedRecord,
|
||||
type BlobManifestRecord,
|
||||
type BootstrapBaselineRecord,
|
||||
type ProjectionNoteRecord
|
||||
} from './factLogFormat.js'
|
||||
import {
|
||||
splitNounMetadataRecord
|
||||
} from '../types/reservedFields.js'
|
||||
import { NounType } from '../types/graphTypes.js'
|
||||
import { v4 as uuidv4 } from '../universal/uuid.js'
|
||||
|
||||
// Swappable msgpack implementation — defaults to the JS codec; a native
|
||||
// provider (registered via the plugin registry's 'msgpack' key) may replace
|
||||
|
|
@ -65,7 +112,12 @@ export function setFactCodec(impl: {
|
|||
export const FACTS_PREFIX = '_generations/facts'
|
||||
/** The facts manifest path (JSON). */
|
||||
export const FACTS_MANIFEST_PATH = `${FACTS_PREFIX}/manifest.json`
|
||||
/** Current segment format version (header field; additive-only within a major). */
|
||||
/**
|
||||
* The v1 segment format version — the MANIFEST's formatVersion gate and the
|
||||
* header value of v1 (minter-less) tails. NOT the live-write ceiling: new
|
||||
* tails write `FACT_LOG_FORMAT_V2` (src/db/factLogFormat.ts) whenever the
|
||||
* int minter is installed; both versions are read forever, per segment.
|
||||
*/
|
||||
export const FACTS_FORMAT_VERSION = 1
|
||||
/** Rotation threshold: seal the tail segment once it exceeds this many bytes. */
|
||||
const SEGMENT_ROTATE_BYTES = 8 * 1024 * 1024
|
||||
|
|
@ -83,6 +135,29 @@ export interface FactOp {
|
|||
record: { metadata: unknown | null; vector: unknown | null } | null
|
||||
}
|
||||
|
||||
/**
|
||||
* V2-native records beyond noun/verb ops that a fact may carry through the
|
||||
* ENCODER (types 6/7/8/9/10 of the v2 registry: embed markers, blob
|
||||
* manifests, projection notes, bootstrap baselines). Encoder-ready by
|
||||
* design; nothing produces them yet — the deferred-embed sidecar and blob
|
||||
* lifecycle remodel onto these records in a later leg.
|
||||
*/
|
||||
export type FactMarkerRecord =
|
||||
| EmbedPendingRecord
|
||||
| EmbedLandedRecord
|
||||
| BlobManifestRecord
|
||||
| ProjectionNoteRecord
|
||||
| BootstrapBaselineRecord
|
||||
|
||||
/**
|
||||
* Mints the dense integer handle for an entity/verb id at fact-append time —
|
||||
* REQUIRED to be reproducible: a rebuilt id mapper must reproduce the same
|
||||
* assignments exactly, so the only legal implementation delegates to the
|
||||
* metadata index's id mapper (`getOrAssign`). Returns a POSITIVE bigint; a
|
||||
* minter that cannot resolve its mapper throws — an int of 0 is never written.
|
||||
*/
|
||||
export type FactIntMinter = (kind: 'noun' | 'verb', id: string) => bigint
|
||||
|
||||
/** One committed generation, as scanned back out of the log. */
|
||||
export interface CommitFact {
|
||||
generation: number
|
||||
|
|
@ -90,6 +165,12 @@ export interface CommitFact {
|
|||
ops: FactOp[]
|
||||
meta?: Record<string, unknown>
|
||||
blobHashes?: string[]
|
||||
/**
|
||||
* V2-native marker records riding this fact (see {@link FactMarkerRecord}).
|
||||
* Optional and additive: absent on every v1 fact and on every fact the
|
||||
* current writers produce; requires a v2 tail to encode.
|
||||
*/
|
||||
records?: FactMarkerRecord[]
|
||||
}
|
||||
|
||||
/** The telemetry a scan batch carries (frozen shape). */
|
||||
|
|
@ -143,6 +224,12 @@ interface FactsManifest {
|
|||
/** The append target. Its true content is established by scanning (crash tolerance). */
|
||||
tailSegment: string | null
|
||||
updatedAt: string
|
||||
/**
|
||||
* This brain's stable id (additive, v2 cutover): minted as a uuid at the
|
||||
* first v2 tail creation and never changed; the `log.genesis` record
|
||||
* carries it. Absent on logs that have never had a v2 tail.
|
||||
*/
|
||||
brainId?: string
|
||||
}
|
||||
|
||||
/** The narrow byte-level storage surface the fact log rides. */
|
||||
|
|
@ -251,40 +338,339 @@ function decodeFact(payload: Uint8Array): CommitFact {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deep-normalize a decoded v2 JSON position (metadata legs, meta maps,
|
||||
* notes) back to plain-JSON values: the v2 codec decodes msgpack int64/uint64
|
||||
* as `bigint` (its u64 wire discipline), but canonical records are JSON — a
|
||||
* metadata timestamp like `createdAt: 1786…` must come back as the NUMBER it
|
||||
* was encoded from. Safe-range bigints narrow exactly; anything beyond the
|
||||
* safe-integer range in a JSON position refuses loudly (it cannot have come
|
||||
* from a JSON write).
|
||||
*/
|
||||
function normalizeWireJson(value: unknown): unknown {
|
||||
if (typeof value === 'bigint') {
|
||||
if (value > BigInt(Number.MAX_SAFE_INTEGER) || value < -BigInt(Number.MAX_SAFE_INTEGER)) {
|
||||
throw new Error(
|
||||
`fact log v2: decoded integer ${value} exceeds the JS safe-integer range in a JSON position`
|
||||
)
|
||||
}
|
||||
return Number(value)
|
||||
}
|
||||
if (Array.isArray(value)) return value.map(normalizeWireJson)
|
||||
if (value && typeof value === 'object' && !(value instanceof Uint8Array)) {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = normalizeWireJson(v)
|
||||
return out
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON-serialization equivalence for a v2 ENCODE-side JSON position: drop
|
||||
* undefined-valued object keys and map undefined array elements to null —
|
||||
* exactly what `JSON.stringify` does when canonical records are persisted.
|
||||
* Commit facts are built from write-cache-WARM objects that may still carry
|
||||
* undefined-valued engine keys (`service: undefined`, …) which the durable
|
||||
* JSON never had; msgpack would preserve them as nil (the v1 capture's known
|
||||
* wart), so the v2 capture — the future storage authority — sanitizes to the
|
||||
* DURABLE truth instead.
|
||||
*/
|
||||
function toJsonSafe(value: unknown): unknown {
|
||||
if (value === undefined) return null
|
||||
if (Array.isArray(value)) return value.map((v) => (v === undefined ? null : toJsonSafe(v)))
|
||||
if (value && typeof value === 'object' && !(value instanceof Uint8Array)) {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) {
|
||||
if (v === undefined) continue
|
||||
out[k] = toJsonSafe(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
/** Mirror of the storage layer's stored-timestamp normalization, minus its
|
||||
* `Date.now()` fallback (a DECODER must be deterministic — an unreadable
|
||||
* timestamp is omitted, and the divergence surfaces via the oracle). */
|
||||
function reconstructTimestamp(value: unknown): number | undefined {
|
||||
if (typeof value === 'number' && value > 0) return value
|
||||
if (
|
||||
value !== null &&
|
||||
typeof value === 'object' &&
|
||||
typeof (value as { seconds?: unknown }).seconds === 'number'
|
||||
) {
|
||||
return (value as { seconds: number }).seconds * 1000
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild a noun's canonical VECTOR-FILE wrapper from a v2 after-image —
|
||||
* the read-side of the hydration law. Canonical noun vector files hold the
|
||||
* denormalized enumerable entity (`{id, vector, connections, level, type,
|
||||
* …reserved fields…, metadata}` — the write path's composition); the v2
|
||||
* record deliberately carries only the ENTITY state (metadata leg + embedding
|
||||
* floats), because connections/level are derived HNSW residue with their own
|
||||
* rebuild paths (empty in every 8.x write) and the denormalized top-level
|
||||
* fields are projections of the metadata leg. This reconstruction applies
|
||||
* the SAME split/hydrate law the storage layer uses
|
||||
* (`splitNounMetadataRecord` — the single source of truth in
|
||||
* src/types/reservedFields.ts; field map mirrors
|
||||
* `BaseStorage.hydrateNounWithMetadata`, undefined keys omitted exactly as
|
||||
* JSON serialization omits them), so in the no-drift case the reconstructed
|
||||
* wrapper digests byte-equal to canonical. A drifted denormalized copy
|
||||
* surfaces as an oracle `state-differs` — named, never silently absorbed.
|
||||
*/
|
||||
function reconstructNounWrapper(
|
||||
id: string,
|
||||
metadataLeg: unknown,
|
||||
floats: number[]
|
||||
): Record<string, unknown> {
|
||||
const { reserved, custom } = splitNounMetadataRecord(
|
||||
(metadataLeg ?? null) as Record<string, unknown> | null
|
||||
)
|
||||
const wrapper: Record<string, unknown> = {
|
||||
id,
|
||||
vector: floats,
|
||||
connections: {},
|
||||
level: 0,
|
||||
type: (reserved.noun as string) || NounType.Thing
|
||||
}
|
||||
if (reserved.subtype !== undefined) wrapper.subtype = reserved.subtype
|
||||
if (reserved.visibility !== undefined) wrapper.visibility = reserved.visibility
|
||||
const createdAt = reconstructTimestamp(reserved.createdAt)
|
||||
if (createdAt !== undefined) wrapper.createdAt = createdAt
|
||||
const updatedAt = reconstructTimestamp(reserved.updatedAt)
|
||||
if (updatedAt !== undefined) wrapper.updatedAt = updatedAt
|
||||
if (reserved.confidence !== undefined) wrapper.confidence = reserved.confidence
|
||||
if (reserved.weight !== undefined) wrapper.weight = reserved.weight
|
||||
if (reserved.service !== undefined) wrapper.service = reserved.service
|
||||
if (reserved.data !== undefined) wrapper.data = reserved.data
|
||||
if (reserved.createdBy !== undefined) wrapper.createdBy = reserved.createdBy
|
||||
wrapper._rev = typeof reserved._rev === 'number' ? reserved._rev : 1
|
||||
wrapper.metadata = custom
|
||||
return wrapper
|
||||
}
|
||||
|
||||
/** Coerce a candidate embedding to `number[]`: plain arrays pass through
|
||||
* (element-checked); numeric typed arrays (the JS HNSW rebuild path stores
|
||||
* `Float32Array` vectors on the memory adapter) widen via `Array.from`. */
|
||||
function floatsOf(candidate: unknown, context: string): number[] | undefined {
|
||||
if (Array.isArray(candidate)) {
|
||||
for (const el of candidate) {
|
||||
if (typeof el !== 'number') {
|
||||
throw new Error(`fact log v2: ${context} vector carries a non-number element`)
|
||||
}
|
||||
}
|
||||
return candidate as number[]
|
||||
}
|
||||
if (ArrayBuffer.isView(candidate) && !(candidate instanceof DataView)) {
|
||||
return Array.from(candidate as unknown as ArrayLike<number>)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** Extract the embedding float array from a canonical vector value: a bare
|
||||
* float array (or numeric typed array) passes through; a wrapper object
|
||||
* yields its `vector` floats; `null` stays `null`; anything else refuses
|
||||
* loudly. */
|
||||
function embeddingLegOf(value: unknown, context: string): number[] | null {
|
||||
if (value === null || value === undefined) return null
|
||||
const direct = floatsOf(value, context)
|
||||
if (direct !== undefined) return direct
|
||||
if (typeof value === 'object') {
|
||||
const nested = floatsOf((value as { vector?: unknown }).vector, context)
|
||||
if (nested !== undefined) return nested
|
||||
}
|
||||
throw new Error(
|
||||
`fact log v2: ${context} has a canonical vector record with no float vector — ` +
|
||||
`cannot encode its after-image`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Map one decoded v2 fact to the {@link CommitFact} shape every consumer
|
||||
* already reads: noun/verb after-images and tombstones become ops (vector
|
||||
* wrappers reconstructed — see {@link reconstructNounWrapper}); a
|
||||
* `batch.meta` record becomes `meta` when the fact position carries none;
|
||||
* `log.genesis` is log-level metadata (its width was verified at decode) and
|
||||
* is not an op; marker records surface on the additive `records` field so
|
||||
* nothing is silently dropped. Decoded JSON positions are normalized back
|
||||
* from the codec's bigint discipline ({@link normalizeWireJson}).
|
||||
*/
|
||||
function commitFactFromV2(f: CommitFactV2): CommitFact {
|
||||
const ops: FactOp[] = []
|
||||
const markers: FactMarkerRecord[] = []
|
||||
let batchMeta: Record<string, unknown> | undefined
|
||||
for (const r of f.records) {
|
||||
switch (r.type) {
|
||||
case 'noun.afterImage': {
|
||||
const metadata = normalizeWireJson(r.metadata) ?? null
|
||||
let vector: unknown | null = null
|
||||
if (r.vectorLeg !== null) {
|
||||
if (!Array.isArray(r.vectorLeg)) {
|
||||
throw new Error(
|
||||
`fact log v2: noun.afterImage ${r.id} carries a vector ref — this reader ` +
|
||||
`resolves inline vectors only (refs are a later leg); refusing`
|
||||
)
|
||||
}
|
||||
vector = reconstructNounWrapper(r.id, metadata, r.vectorLeg)
|
||||
}
|
||||
ops.push({ kind: 'noun', id: r.id, record: { metadata, vector } })
|
||||
break
|
||||
}
|
||||
case 'noun.tombstone':
|
||||
ops.push({ kind: 'noun', id: r.id, record: null })
|
||||
break
|
||||
case 'verb.afterImage': {
|
||||
const metadata = normalizeWireJson(r.metadata) ?? null
|
||||
if (r.vectorLeg !== null && !Array.isArray(r.vectorLeg)) {
|
||||
throw new Error(
|
||||
`fact log v2: verb.afterImage ${r.id} carries a vector ref — this reader ` +
|
||||
`resolves inline vectors only (refs are a later leg); refusing`
|
||||
)
|
||||
}
|
||||
// The canonical verb vector-file wrapper: endpoints + verb name ride
|
||||
// as first-class v2 wire fields precisely so this reconstruction is
|
||||
// exact ({id, vector, connections:{}, verb, sourceId, targetId} —
|
||||
// verbs carry no `level`).
|
||||
const vector: Record<string, unknown> = {
|
||||
id: r.id,
|
||||
vector: r.vectorLeg ?? [],
|
||||
connections: {},
|
||||
verb: r.verb,
|
||||
sourceId: r.sourceId,
|
||||
targetId: r.targetId
|
||||
}
|
||||
ops.push({ kind: 'verb', id: r.id, record: { metadata, vector } })
|
||||
break
|
||||
}
|
||||
case 'verb.tombstone':
|
||||
ops.push({ kind: 'verb', id: r.id, record: null })
|
||||
break
|
||||
case 'batch.meta':
|
||||
batchMeta = normalizeWireJson(r.meta) as Record<string, unknown>
|
||||
break
|
||||
case 'log.genesis':
|
||||
break // the log's birth certificate — log-level metadata, not an op
|
||||
case 'projection.note':
|
||||
markers.push({ ...r, note: normalizeWireJson(r.note) as Record<string, unknown> })
|
||||
break
|
||||
case 'bootstrap.baseline':
|
||||
markers.push({ ...r, metadata: normalizeWireJson(r.metadata) })
|
||||
break
|
||||
default:
|
||||
// embed.pending / embed.landed / blob.manifest carry no loose JSON maps.
|
||||
markers.push(r)
|
||||
break
|
||||
}
|
||||
}
|
||||
const meta = f.meta ? (normalizeWireJson(f.meta) as Record<string, unknown>) : batchMeta
|
||||
return {
|
||||
generation: f.generation,
|
||||
timestamp: f.timestamp,
|
||||
ops,
|
||||
...(meta ? { meta } : {}),
|
||||
...(f.blobHashes && f.blobHashes.length > 0 ? { blobHashes: f.blobHashes } : {}),
|
||||
...(markers.length > 0 ? { records: markers } : {})
|
||||
}
|
||||
}
|
||||
|
||||
/** One intact v2 frame's extent inside a segment (byte-slicing support). */
|
||||
interface V2FrameExtent {
|
||||
/** Byte offset just past this frame. */
|
||||
end: number
|
||||
/** The frame's generation (0 for pad filler). */
|
||||
generation: number
|
||||
/** True when the frame is a pad (invisible filler). */
|
||||
isPad: boolean
|
||||
}
|
||||
|
||||
/** Walk a v2 segment's intact frames (torn-tail terminated), returning each
|
||||
* frame's extent — the byte-level view truncation slices against, so kept
|
||||
* frames are never re-encoded (byte-immutability of CRC-covered frames). */
|
||||
function walkV2Frames(bytes: Uint8Array): V2FrameExtent[] {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
||||
const extents: V2FrameExtent[] = []
|
||||
let offset = HEADER_BYTES
|
||||
while (offset + FRAME_PREFIX_BYTES <= bytes.length) {
|
||||
const length = view.getUint32(offset, true)
|
||||
const expectedCrc = view.getUint32(offset + 4, true)
|
||||
const start = offset + FRAME_PREFIX_BYTES
|
||||
const end = start + length
|
||||
if (end > bytes.length) break // torn tail
|
||||
const payload = bytes.subarray(start, end)
|
||||
if (crc32c(payload) !== expectedCrc) break // torn tail
|
||||
const fact = decodeFormatFact(payload, FACT_LOG_FORMAT_V2, {
|
||||
expectedIdSpaceWidth: 64
|
||||
}) as CommitFactV2
|
||||
extents.push({ end, generation: fact.generation, isPad: fact.records.length === 0 })
|
||||
offset = end
|
||||
}
|
||||
return extents
|
||||
}
|
||||
|
||||
/**
|
||||
* The byte offset a v2 segment is cut at to keep exactly the facts with
|
||||
* `generation ≤ keepThrough`: the end of the last kept FACT frame (pads
|
||||
* between kept facts sit inside the retained span; pads after the cut are
|
||||
* dropped and re-sealed at the next sync). When nothing is dropped the cut
|
||||
* lands after the last intact frame — trailing pads retained, only a torn
|
||||
* suffix (if any) removed.
|
||||
*/
|
||||
function v2CutOffset(extents: V2FrameExtent[], keepThrough: number): number {
|
||||
let cut = HEADER_BYTES
|
||||
let lastIntactEnd = HEADER_BYTES
|
||||
for (const e of extents) {
|
||||
lastIntactEnd = e.end
|
||||
if (e.isPad) continue
|
||||
if (e.generation <= keepThrough) {
|
||||
cut = e.end
|
||||
} else {
|
||||
return cut // first beyond-keep fact: everything from here (pads included) goes
|
||||
}
|
||||
}
|
||||
return lastIntactEnd
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a segment's bytes: verify the header, then walk frames until the end
|
||||
* or a torn tail (length overrun / CRC mismatch), which terminates the walk —
|
||||
* everything before it is intact. Returns the decoded facts plus the byte
|
||||
* length of the VALID prefix (header + intact frames), which reconciliation
|
||||
* uses to cut a torn tail without re-encoding.
|
||||
* everything before it is intact. The header's formatVersion selects the
|
||||
* decoder: the v1 walk below is byte-identical to the original v1 reader;
|
||||
* v2 segments decode through the reference codec (`decodeGroupV2`, pads
|
||||
* invisible, id-space width verified at 64 — a disagreeing genesis throws
|
||||
* the codec's typed `GenesisWidthMismatchError`). Returns the decoded facts
|
||||
* plus the byte length of the VALID prefix (header + intact frames), which
|
||||
* reconciliation uses to cut a torn tail without re-encoding.
|
||||
*/
|
||||
function parseSegment(
|
||||
file: string,
|
||||
bytes: Uint8Array
|
||||
): { facts: CommitFact[]; validBytes: number } {
|
||||
): { facts: CommitFact[]; validBytes: number; formatVersion: number; sealSize?: number } {
|
||||
if (bytes.length < HEADER_BYTES) {
|
||||
prodLog.warn(`[FactLog] segment ${file} shorter than its header — treating as empty`)
|
||||
return { facts: [], validBytes: 0 }
|
||||
return { facts: [], validBytes: 0, formatVersion: 0 }
|
||||
}
|
||||
for (let i = 0; i < MAGIC.length; i++) {
|
||||
if (bytes[i] !== MAGIC[i]) {
|
||||
throw new Error(`fact log: segment ${file} has a bad magic — not a fact segment`)
|
||||
let header: { formatVersion: number; sealSize?: number }
|
||||
try {
|
||||
header = parseSegmentHeader(bytes.subarray(0, HEADER_BYTES))
|
||||
} catch (err) {
|
||||
throw new Error(`fact log: segment ${file}: ${(err as Error).message}`)
|
||||
}
|
||||
}
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
||||
const version = view.getUint32(8, true)
|
||||
if (version !== FACTS_FORMAT_VERSION) {
|
||||
throw new Error(
|
||||
`fact log: segment ${file} has formatVersion ${version}; this build reads ${FACTS_FORMAT_VERSION}`
|
||||
)
|
||||
}
|
||||
for (let i = 20; i < HEADER_BYTES; i++) {
|
||||
if (bytes[i] !== 0) {
|
||||
// Non-zero reserved bytes = a future format this build cannot verify.
|
||||
throw new Error(`fact log: segment ${file} has non-zero reserved header bytes — unverifiable`)
|
||||
|
||||
if (header.formatVersion === FACT_LOG_FORMAT_V2) {
|
||||
const group = decodeGroupV2(bytes.subarray(HEADER_BYTES), { expectedIdSpaceWidth: 64 })
|
||||
return {
|
||||
facts: group.facts.map(commitFactFromV2),
|
||||
validBytes: HEADER_BYTES + group.validBytes,
|
||||
formatVersion: FACT_LOG_FORMAT_V2,
|
||||
sealSize: header.sealSize
|
||||
}
|
||||
}
|
||||
|
||||
// v1 walk — byte-identical to the original reader.
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
||||
const facts: CommitFact[] = []
|
||||
let offset = HEADER_BYTES
|
||||
while (offset + FRAME_PREFIX_BYTES <= bytes.length) {
|
||||
|
|
@ -298,7 +684,7 @@ function parseSegment(
|
|||
facts.push(decodeFact(payload))
|
||||
offset = end
|
||||
}
|
||||
return { facts, validBytes: offset }
|
||||
return { facts, validBytes: offset, formatVersion: FACT_LOG_FORMAT_V1 }
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -318,18 +704,38 @@ export class FactLog {
|
|||
}
|
||||
/** Decoded facts of the TAIL segment (bounded by the rotation threshold). */
|
||||
private tailFacts: CommitFact[] = []
|
||||
/** Byte size of the tail segment file (valid prefix). */
|
||||
/** Byte size of the tail segment file (valid prefix, pads included —
|
||||
* pads count toward bytes but NEVER toward facts). */
|
||||
private tailBytes = 0
|
||||
/** Highest generation in the log (0 = empty). */
|
||||
private head = 0
|
||||
/** Segment paths appended since the last sync (the fsync batch). */
|
||||
private readonly dirtySegments = new Set<string>()
|
||||
/** The TAIL segment's on-disk format version (selects the live encoder). */
|
||||
private tailVersion: number = FACT_LOG_FORMAT_V1
|
||||
/** The tail's sector-seal size (v2 tails; from its header on reopen). */
|
||||
private tailSealSize: number = DEFAULT_SEAL_SIZE
|
||||
/** The v2 int minter (see {@link FactIntMinter}); null = v1 live writes. */
|
||||
private intMinter: FactIntMinter | null = null
|
||||
|
||||
constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) {
|
||||
this.storage = storage
|
||||
this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES
|
||||
}
|
||||
|
||||
/**
|
||||
* Install the v2 int minter — the capability gate for v2 LIVE WRITES.
|
||||
* With a minter installed, every NEW tail segment writes the v2 format and
|
||||
* after-image records carry minted dense ints; without one, live writes
|
||||
* stay v1 (no authority could reproduce int assignments, and 0 is never
|
||||
* written). The brain wires this from the metadata index's id mapper right
|
||||
* after the index is ready; an existing v1 tail cuts over on the next
|
||||
* append (empty tail: re-headed in place; non-empty: sealed by rotation).
|
||||
*/
|
||||
setIntMinter(mint: FactIntMinter): void {
|
||||
this.intMinter = mint
|
||||
}
|
||||
|
||||
/** The highest committed generation the log holds (0 = empty). */
|
||||
headGeneration(): number {
|
||||
return this.head
|
||||
|
|
@ -412,11 +818,17 @@ export class FactLog {
|
|||
const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}`
|
||||
const bytes = await this.storage.readRawBytes(tailPath)
|
||||
if (bytes === null) {
|
||||
// Manifest named a tail whose first byte never landed — an empty tail.
|
||||
// Manifest named a tail whose first byte never landed — an empty
|
||||
// tail. Its header (and format version) is established at the next
|
||||
// append (see the tail-provisioning ladder there).
|
||||
this.tailFacts = []
|
||||
this.tailBytes = 0
|
||||
} else {
|
||||
const { facts, validBytes } = parseSegment(this.manifest.tailSegment, bytes)
|
||||
const parsed = parseSegment(this.manifest.tailSegment, bytes)
|
||||
const { facts, validBytes } = parsed
|
||||
this.tailVersion =
|
||||
parsed.formatVersion === FACT_LOG_FORMAT_V2 ? FACT_LOG_FORMAT_V2 : FACT_LOG_FORMAT_V1
|
||||
this.tailSealSize = parsed.sealSize ?? DEFAULT_SEAL_SIZE
|
||||
const kept = facts.filter((f) => f.generation <= committedGeneration)
|
||||
if (kept.length !== facts.length || validBytes !== bytes.length) {
|
||||
const dropped = facts.length - kept.length
|
||||
|
|
@ -426,7 +838,16 @@ export class FactLog {
|
|||
`${committedGeneration} from the tail (never committed)`
|
||||
)
|
||||
}
|
||||
if (this.tailVersion === FACT_LOG_FORMAT_V2) {
|
||||
// V2: byte-slice at frame boundaries — CRC-covered frames are
|
||||
// byte-immutable; a truncation never re-encodes what it keeps.
|
||||
const cut = v2CutOffset(walkV2Frames(bytes), committedGeneration)
|
||||
await this.storage.writeRawBytes(tailPath, bytes.subarray(0, cut))
|
||||
this.tailFacts = kept
|
||||
this.tailBytes = cut
|
||||
} else {
|
||||
await this.rewriteTail(kept)
|
||||
}
|
||||
} else {
|
||||
this.tailFacts = facts
|
||||
this.tailBytes = validBytes
|
||||
|
|
@ -441,6 +862,15 @@ export class FactLog {
|
|||
* Append one committed generation's fact. NOT durable until {@link sync} —
|
||||
* the caller batches durability at its commit barrier (transact syncs in
|
||||
* the same call; Model-B group-commit syncs at flush).
|
||||
*
|
||||
* Tail provisioning (in order): a missing tail starts one; a named tail
|
||||
* whose header never landed (manifest-first crash) gets its header now; an
|
||||
* existing V1 tail cuts over to v2 once the minter is installed (empty:
|
||||
* re-headed in place, non-empty: sealed by rotation — v1 segments are never
|
||||
* rewritten); a full tail rotates. The frame then encodes in the TAIL's
|
||||
* format: v2 tails carry after-image records with minted ints (and the
|
||||
* genesis record on the very first fact of a brand-new log); v1 tails keep
|
||||
* the v1 wire format byte-identically.
|
||||
*/
|
||||
async append(fact: CommitFact): Promise<void> {
|
||||
if (fact.generation <= this.head) {
|
||||
|
|
@ -450,10 +880,42 @@ export class FactLog {
|
|||
}
|
||||
if (this.manifest.tailSegment === null) {
|
||||
await this.startTail(fact.generation)
|
||||
} else if (this.tailBytes === 0) {
|
||||
await this.reinitializeTailHeader()
|
||||
} else if (this.intMinter !== null && this.tailVersion === FACT_LOG_FORMAT_V1) {
|
||||
if (this.tailFacts.length === 0 && this.tailBytes <= HEADER_BYTES) {
|
||||
await this.upgradeEmptyTailToV2()
|
||||
} else {
|
||||
await this.rotate(fact.generation)
|
||||
}
|
||||
} else if (this.tailBytes >= this.rotateBytes) {
|
||||
await this.rotate(fact.generation)
|
||||
}
|
||||
const frame = encodeFrame(fact)
|
||||
|
||||
let frame: Uint8Array
|
||||
if (this.tailVersion === FACT_LOG_FORMAT_V2) {
|
||||
const records = this.buildV2Records(fact)
|
||||
if (this.needsGenesis()) {
|
||||
if (this.ensureBrainId()) await this.persistManifest()
|
||||
records.unshift(this.genesisRecord())
|
||||
}
|
||||
frame = encodeFactV2({
|
||||
generation: fact.generation,
|
||||
timestamp: fact.timestamp,
|
||||
records,
|
||||
...(fact.meta ? { meta: toJsonSafe(fact.meta) as Record<string, unknown> } : {}),
|
||||
...(fact.blobHashes && fact.blobHashes.length > 0 ? { blobHashes: fact.blobHashes } : {})
|
||||
})
|
||||
} else {
|
||||
if (fact.records && fact.records.length > 0) {
|
||||
throw new Error(
|
||||
`fact log: marker records (${fact.records.map((r) => r.type).join(', ')}) require a ` +
|
||||
`v2 tail segment — this log's tail is v1 (no int minter installed); refusing rather ` +
|
||||
`than silently dropping them`
|
||||
)
|
||||
}
|
||||
frame = encodeFrame(fact)
|
||||
}
|
||||
const tailPath = `${FACTS_PREFIX}/${this.manifest.tailSegment}`
|
||||
await this.storage.appendRawBytes(tailPath, frame)
|
||||
this.tailFacts.push(fact)
|
||||
|
|
@ -462,8 +924,16 @@ export class FactLog {
|
|||
this.dirtySegments.add(tailPath)
|
||||
}
|
||||
|
||||
/** Fsync every segment appended since the last sync. */
|
||||
/**
|
||||
* Fsync every segment appended since the last sync. SEALS AT SYNC: a v2
|
||||
* tail is first padded to its sector-seal boundary (one pad frame,
|
||||
* invisible to readers; a gap smaller than the smallest constructible pad
|
||||
* frame pads through one extra sector — the codec's rule), so every
|
||||
* durability barrier leaves the tail sector-aligned: a torn page can only
|
||||
* tear INSIDE the group being written, never a previously-sealed one.
|
||||
*/
|
||||
async sync(): Promise<void> {
|
||||
await this.padTailToSealBoundary()
|
||||
if (this.dirtySegments.size === 0) return
|
||||
const paths = [...this.dirtySegments]
|
||||
this.dirtySegments.clear()
|
||||
|
|
@ -671,7 +1141,25 @@ export class FactLog {
|
|||
`(head ${this.head}) — the fact to drop was already sealed; the log needs reopen`
|
||||
)
|
||||
}
|
||||
if (this.tailVersion === FACT_LOG_FORMAT_V2) {
|
||||
// V2: byte-slice at frame boundaries (kept frames stay byte-identical;
|
||||
// pads between kept facts are retained inside the prefix, trailing pads
|
||||
// go and the next sync re-seals). The dropped frames may be unsynced —
|
||||
// readRawBytes is read-after-write coherent over the append path.
|
||||
const file = this.manifest.tailSegment
|
||||
if (!file) return
|
||||
const tailPath = `${FACTS_PREFIX}/${file}`
|
||||
const bytes = await this.storage.readRawBytes(tailPath)
|
||||
if (bytes === null) {
|
||||
throw new Error(`fact log: dropAbove(${keepThrough}) cannot read the tail segment ${file}`)
|
||||
}
|
||||
const cut = v2CutOffset(walkV2Frames(bytes), keepThrough)
|
||||
await this.storage.writeRawBytes(tailPath, bytes.subarray(0, cut))
|
||||
this.tailFacts = kept
|
||||
this.tailBytes = cut
|
||||
} else {
|
||||
await this.rewriteTail(kept)
|
||||
}
|
||||
this.head = this.computeHead()
|
||||
}
|
||||
|
||||
|
|
@ -684,25 +1172,42 @@ export class FactLog {
|
|||
return 0
|
||||
}
|
||||
|
||||
/** The header bytes for a NEW tail: v2 whenever the minter is installed. */
|
||||
private newTailHeader(firstGeneration: number): Uint8Array {
|
||||
return this.intMinter !== null
|
||||
? encodeSegmentHeaderV2(firstGeneration, DEFAULT_SEAL_SIZE)
|
||||
: buildHeader(firstGeneration)
|
||||
}
|
||||
|
||||
/** Record the just-created tail's format in memory (mirrors its header). */
|
||||
private noteFreshTail(): void {
|
||||
this.tailVersion = this.intMinter !== null ? FACT_LOG_FORMAT_V2 : FACT_LOG_FORMAT_V1
|
||||
this.tailSealSize = DEFAULT_SEAL_SIZE
|
||||
}
|
||||
|
||||
/** Create the very first tail segment (manifest-first, then header bytes). */
|
||||
private async startTail(firstGeneration: number): Promise<void> {
|
||||
const file = segmentFileName(firstGeneration)
|
||||
this.manifest.tailSegment = file
|
||||
if (this.intMinter !== null) this.ensureBrainId()
|
||||
await this.persistManifest()
|
||||
await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, buildHeader(firstGeneration))
|
||||
await this.storage.appendRawBytes(`${FACTS_PREFIX}/${file}`, this.newTailHeader(firstGeneration))
|
||||
this.tailFacts = []
|
||||
this.tailBytes = HEADER_BYTES
|
||||
this.noteFreshTail()
|
||||
}
|
||||
|
||||
/**
|
||||
* Seal the tail into the manifest and start a new one. Manifest-first: the
|
||||
* flip both seals the old tail AND names the new one atomically, so no
|
||||
* segment file ever exists unaccounted for.
|
||||
* segment file ever exists unaccounted for. The NEW tail's format follows
|
||||
* the minter gate ({@link newTailHeader}) — this is also the v1→v2 cutover
|
||||
* seam for a non-empty v1 tail (sealed as-is, never rewritten).
|
||||
*/
|
||||
private async rotate(nextGeneration: number): Promise<void> {
|
||||
const sealedFile = this.manifest.tailSegment
|
||||
if (!sealedFile) return
|
||||
// Seal what the tail actually holds.
|
||||
// Seal what the tail actually holds (sync() also sector-seals a v2 tail).
|
||||
await this.sync() // sealed segments are always fully durable
|
||||
const entry: SegmentEntry = {
|
||||
file: sealedFile,
|
||||
|
|
@ -714,10 +1219,181 @@ export class FactLog {
|
|||
const newFile = segmentFileName(nextGeneration)
|
||||
this.manifest.segments.push(entry)
|
||||
this.manifest.tailSegment = newFile
|
||||
if (this.intMinter !== null) this.ensureBrainId()
|
||||
await this.persistManifest()
|
||||
await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, buildHeader(nextGeneration))
|
||||
await this.storage.appendRawBytes(`${FACTS_PREFIX}/${newFile}`, this.newTailHeader(nextGeneration))
|
||||
this.tailFacts = []
|
||||
this.tailBytes = HEADER_BYTES
|
||||
this.noteFreshTail()
|
||||
}
|
||||
|
||||
/**
|
||||
* The v1→v2 cutover for an EMPTY v1 tail: re-head it in place (nothing but
|
||||
* the 32-byte header exists, so no v1 frame is ever rewritten). Also the
|
||||
* cheapest cutover shape: brand-new brains whose first tail predates the
|
||||
* minter installation converge here on their first post-install append.
|
||||
*/
|
||||
private async upgradeEmptyTailToV2(): Promise<void> {
|
||||
const file = this.manifest.tailSegment
|
||||
if (!file) return
|
||||
if (this.ensureBrainId()) await this.persistManifest()
|
||||
const first = this.segmentFirstGenerationFromName(file)
|
||||
const path = `${FACTS_PREFIX}/${file}`
|
||||
await this.storage.writeRawBytes(path, encodeSegmentHeaderV2(first, DEFAULT_SEAL_SIZE))
|
||||
this.tailBytes = HEADER_BYTES
|
||||
this.tailVersion = FACT_LOG_FORMAT_V2
|
||||
this.tailSealSize = DEFAULT_SEAL_SIZE
|
||||
this.dirtySegments.add(path)
|
||||
}
|
||||
|
||||
/**
|
||||
* A manifest-named tail whose header never landed (crash between the
|
||||
* manifest flip and the first header byte — previously this appended
|
||||
* frames into a headerless file the next open could not parse): write the
|
||||
* header now, in the CURRENT format gate.
|
||||
*/
|
||||
private async reinitializeTailHeader(): Promise<void> {
|
||||
const file = this.manifest.tailSegment
|
||||
if (!file) return
|
||||
if (this.intMinter !== null && this.ensureBrainId()) await this.persistManifest()
|
||||
const first = this.segmentFirstGenerationFromName(file)
|
||||
const path = `${FACTS_PREFIX}/${file}`
|
||||
await this.storage.writeRawBytes(path, this.newTailHeader(first))
|
||||
this.tailBytes = HEADER_BYTES
|
||||
this.noteFreshTail()
|
||||
this.dirtySegments.add(path)
|
||||
}
|
||||
|
||||
/** True when the NEXT appended fact is the first fact of a brand-new v2
|
||||
* log — the one that must open with the log.genesis record. */
|
||||
private needsGenesis(): boolean {
|
||||
return (
|
||||
this.tailVersion === FACT_LOG_FORMAT_V2 &&
|
||||
this.manifest.segments.length === 0 &&
|
||||
this.tailFacts.length === 0
|
||||
)
|
||||
}
|
||||
|
||||
/** Mint the brain id into the manifest if absent; true when it changed. */
|
||||
private ensureBrainId(): boolean {
|
||||
if (this.manifest.brainId) return false
|
||||
this.manifest.brainId = uuidv4()
|
||||
return true
|
||||
}
|
||||
|
||||
/** The log's birth certificate (id-space width 64 — the only width this
|
||||
* writer mints; a reader expecting another width refuses at decode). */
|
||||
private genesisRecord(): LogRecord {
|
||||
const brainId = this.manifest.brainId
|
||||
if (!brainId) {
|
||||
throw new Error(
|
||||
'fact log v2: genesis requires a brainId in the facts manifest — invariant violated'
|
||||
)
|
||||
}
|
||||
return { type: 'log.genesis', idSpaceWidth: 64, brainId, createdAt: Date.now() }
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert one CommitFact's ops (+ optional marker records) to v2 wire
|
||||
* records, MINTING ints at append time: entity/verb ints come from the
|
||||
* injected minter (the metadata index's id mapper — the one authority a
|
||||
* rebuild reproduces exactly). Verb endpoints and the verb name ride as
|
||||
* first-class wire fields, lifted from the canonical verb vector wrapper.
|
||||
* Every refusal here is loud — an after-image without a mintable int, a
|
||||
* verb without endpoints, or a vector record without floats fails the
|
||||
* WRITE, never writes a 0.
|
||||
*/
|
||||
private buildV2Records(fact: CommitFact): LogRecord[] {
|
||||
const mint = (kind: 'noun' | 'verb', id: string): bigint => {
|
||||
if (this.intMinter === null) {
|
||||
throw new Error(
|
||||
`fact log v2: no int minter is installed — cannot mint the ${kind} int for ${id}; ` +
|
||||
`refusing to write a v2 after-image (an int of 0 is never written)`
|
||||
)
|
||||
}
|
||||
const minted = this.intMinter(kind, id)
|
||||
if (typeof minted !== 'bigint' || minted <= 0n) {
|
||||
throw new Error(
|
||||
`fact log v2: the int minter returned ${String(minted)} for ${kind} ${id} — ` +
|
||||
`minted ints are positive bigints; refusing to write`
|
||||
)
|
||||
}
|
||||
return minted
|
||||
}
|
||||
|
||||
const records: LogRecord[] = []
|
||||
for (const op of fact.ops) {
|
||||
if (op.kind === 'noun') {
|
||||
if (op.record === null) {
|
||||
records.push({ type: 'noun.tombstone', id: op.id })
|
||||
continue
|
||||
}
|
||||
records.push({
|
||||
type: 'noun.afterImage',
|
||||
id: op.id,
|
||||
entityInt: mint('noun', op.id),
|
||||
metadata: toJsonSafe(op.record.metadata ?? null),
|
||||
vectorLeg: embeddingLegOf(op.record.vector, `noun ${op.id}`)
|
||||
})
|
||||
} else {
|
||||
if (op.record === null) {
|
||||
records.push({ type: 'verb.tombstone', id: op.id })
|
||||
continue
|
||||
}
|
||||
const wrapper = op.record.vector as Record<string, unknown> | null
|
||||
const verbName = wrapper?.verb
|
||||
const sourceId = wrapper?.sourceId
|
||||
const targetId = wrapper?.targetId
|
||||
if (
|
||||
typeof verbName !== 'string' ||
|
||||
typeof sourceId !== 'string' ||
|
||||
typeof targetId !== 'string'
|
||||
) {
|
||||
throw new Error(
|
||||
`fact log v2: verb ${op.id} has no canonical endpoints (verb/sourceId/targetId ` +
|
||||
`live in its vector record, which is missing or torn) — refusing to write an ` +
|
||||
`after-image that could not be replayed`
|
||||
)
|
||||
}
|
||||
const floats = floatsOf(wrapper?.vector, `verb ${op.id}`) ?? []
|
||||
records.push({
|
||||
type: 'verb.afterImage',
|
||||
id: op.id,
|
||||
verbInt: mint('verb', op.id),
|
||||
metadata: toJsonSafe(op.record.metadata ?? null),
|
||||
vectorLeg: floats,
|
||||
verb: verbName,
|
||||
sourceId,
|
||||
sourceInt: mint('noun', sourceId),
|
||||
targetId,
|
||||
targetInt: mint('noun', targetId)
|
||||
})
|
||||
}
|
||||
}
|
||||
for (const marker of fact.records ?? []) records.push(marker)
|
||||
return records
|
||||
}
|
||||
|
||||
/**
|
||||
* Pad a v2 tail to its next sector-seal boundary with ONE pad frame —
|
||||
* called from {@link sync} so alignment holds at every durability barrier.
|
||||
* Pads count toward {@link tailBytes} but never toward facts (they are
|
||||
* invisible to every reader); a gap smaller than the smallest constructible
|
||||
* pad frame pads through one extra sector (the codec's rule). No-op for v1
|
||||
* tails, empty tails, and already-aligned tails.
|
||||
*/
|
||||
private async padTailToSealBoundary(): Promise<void> {
|
||||
if (this.tailVersion !== FACT_LOG_FORMAT_V2) return
|
||||
const file = this.manifest.tailSegment
|
||||
if (!file || this.tailBytes <= HEADER_BYTES) return
|
||||
const remainder = this.tailBytes % this.tailSealSize
|
||||
if (remainder === 0) return
|
||||
let padBytes = this.tailSealSize - remainder
|
||||
if (padBytes < minPadFrameBytes()) padBytes += this.tailSealSize
|
||||
const tailPath = `${FACTS_PREFIX}/${file}`
|
||||
await this.storage.appendRawBytes(tailPath, encodePadFrame(padBytes))
|
||||
this.tailBytes += padBytes
|
||||
this.dirtySegments.add(tailPath)
|
||||
}
|
||||
|
||||
/** Atomically persist the manifest (write-new → fsync → rename downstream). */
|
||||
|
|
@ -746,17 +1422,24 @@ export class FactLog {
|
|||
this.tailBytes = total
|
||||
}
|
||||
|
||||
/** Cut a SEALED segment back to `committedGeneration` (atomic replace). */
|
||||
/** Cut a SEALED segment back to `committedGeneration` (atomic replace).
|
||||
* v2 segments byte-slice at frame boundaries (kept frames — pads
|
||||
* included — are never re-encoded); the v1 re-encode path is unchanged. */
|
||||
private async truncateSegmentTo(file: string, committedGeneration: number): Promise<void> {
|
||||
const path = `${FACTS_PREFIX}/${file}`
|
||||
const bytes = await this.storage.readRawBytes(path)
|
||||
if (bytes === null) return
|
||||
const { facts } = parseSegment(file, bytes)
|
||||
const { facts, formatVersion } = parseSegment(file, bytes)
|
||||
const kept = facts.filter((f) => f.generation <= committedGeneration)
|
||||
prodLog.warn(
|
||||
`[FactLog] truncating sealed segment ${file} to generation ${committedGeneration} ` +
|
||||
`(${facts.length - kept.length} uncommitted fact(s) dropped)`
|
||||
)
|
||||
if (formatVersion === FACT_LOG_FORMAT_V2) {
|
||||
const cut = v2CutOffset(walkV2Frames(bytes), committedGeneration)
|
||||
await this.storage.writeRawBytes(path, bytes.subarray(0, cut))
|
||||
return
|
||||
}
|
||||
const first = kept[0]?.generation ?? this.segmentFirstGenerationFromName(file)
|
||||
const parts: Uint8Array[] = [buildHeader(first)]
|
||||
for (const f of kept) parts.push(encodeFrame(f))
|
||||
|
|
|
|||
|
|
@ -26,9 +26,21 @@
|
|||
* position 2 is `records`, not v1's `ops`)
|
||||
*
|
||||
* fact := [ generation:u64, timestamp:u64, records, meta|nil, blobHashes|nil ]
|
||||
* record := [ recordType:u8, recordVersion:u8, ...type-specific fields ]
|
||||
* record := [ recordType:u8, recordVersion:u8, cipherFlag:u8, keyId:bin16|nil,
|
||||
* ...type-specific fields ]
|
||||
*
|
||||
* Record type registry (all recordVersion = 1):
|
||||
* `cipherFlag`/`keyId` are RESERVED crypto envelope fields: `0`/`nil` (a
|
||||
* plaintext record) is the ONLY legal combination this release writes or
|
||||
* reads. Any nonzero cipherFlag or non-nil keyId refuses with the typed
|
||||
* {@link UnknownLogRecordError} ("encrypted records need a newer reader") —
|
||||
* so record-level encryption can land later without a format-version bump on
|
||||
* the one compat surface. No crypto logic exists here; the bytes are reserved
|
||||
* only. Pad records (type 0) are exempt: they are skipped WHOLESALE as
|
||||
* length-only filler, so their fields beyond [type, version] are never
|
||||
* inspected (this keeps pad frames byte-stable across the envelope change).
|
||||
*
|
||||
* Record type registry (all recordVersion = 1; type-specific fields listed —
|
||||
* every record carries the 4-field envelope above first):
|
||||
*
|
||||
* 0 pad [] — length-only filler; readers SKIP; crc-covered
|
||||
* 1 noun.afterImage [id bin16, entityInt u64, metadata, vectorLeg]
|
||||
|
|
@ -109,6 +121,13 @@ export const DEFAULT_SEAL_SIZE = 4096
|
|||
/** The record version this reader knows (all registry types are version 1). */
|
||||
export const LOG_RECORD_VERSION = 1
|
||||
|
||||
/**
|
||||
* The only legal `cipherFlag` value this release: plaintext. The encoder
|
||||
* always writes it (with a nil keyId); the decoder refuses anything else
|
||||
* with {@link UnknownLogRecordError} — encrypted records need a newer reader.
|
||||
*/
|
||||
export const LOG_RECORD_CIPHER_PLAINTEXT = 0
|
||||
|
||||
/** The v2 record-type registry — wire codes for every record type. */
|
||||
export const LOG_RECORD_TYPES = {
|
||||
PAD: 0,
|
||||
|
|
@ -648,22 +667,31 @@ function decodeVectorLeg(wire: unknown, context: string): VectorLeg {
|
|||
// Record encode/decode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Encode one record into its positional wire array. */
|
||||
/**
|
||||
* Encode one record into its positional wire array. Every record leads with
|
||||
* the 4-field envelope [type, version, cipherFlag, keyId]; this release
|
||||
* writes cipherFlag {@link LOG_RECORD_CIPHER_PLAINTEXT} and a nil keyId
|
||||
* always (the fields are crypto-RESERVED, carrying no logic yet).
|
||||
*/
|
||||
function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefined): unknown[] {
|
||||
const T = LOG_RECORD_TYPES
|
||||
const V = LOG_RECORD_VERSION
|
||||
const C = LOG_RECORD_CIPHER_PLAINTEXT
|
||||
const K = null // keyId: nil until record-level encryption exists
|
||||
switch (record.type) {
|
||||
case 'noun.afterImage':
|
||||
return [
|
||||
T.NOUN_AFTER_IMAGE,
|
||||
V,
|
||||
C,
|
||||
K,
|
||||
uuidToBytes(record.id),
|
||||
toWireU64(record.entityInt, 'entityInt'),
|
||||
record.metadata ?? null,
|
||||
encodeVectorLeg(record.vectorLeg, options, `noun.afterImage ${record.id}`)
|
||||
]
|
||||
case 'noun.tombstone':
|
||||
return [T.NOUN_TOMBSTONE, V, uuidToBytes(record.id)]
|
||||
return [T.NOUN_TOMBSTONE, V, C, K, uuidToBytes(record.id)]
|
||||
case 'verb.afterImage': {
|
||||
if (typeof record.verb !== 'string' || record.verb.length === 0) {
|
||||
throw new Error(`fact log v2: verb.afterImage ${record.id} needs a non-empty verb name`)
|
||||
|
|
@ -671,6 +699,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
|
|||
return [
|
||||
T.VERB_AFTER_IMAGE,
|
||||
V,
|
||||
C,
|
||||
K,
|
||||
uuidToBytes(record.id),
|
||||
toWireU64(record.verbInt, 'verbInt'),
|
||||
record.metadata ?? null,
|
||||
|
|
@ -683,16 +713,18 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
|
|||
]
|
||||
}
|
||||
case 'verb.tombstone':
|
||||
return [T.VERB_TOMBSTONE, V, uuidToBytes(record.id)]
|
||||
return [T.VERB_TOMBSTONE, V, C, K, uuidToBytes(record.id)]
|
||||
case 'batch.meta':
|
||||
if (!isPlainMap(record.meta)) {
|
||||
throw new Error('fact log v2: batch.meta requires a map')
|
||||
}
|
||||
return [T.BATCH_META, V, record.meta]
|
||||
return [T.BATCH_META, V, C, K, record.meta]
|
||||
case 'embed.pending':
|
||||
return [
|
||||
T.EMBED_PENDING,
|
||||
V,
|
||||
C,
|
||||
K,
|
||||
uuidToBytes(record.id),
|
||||
toWireU64(record.enqueuedAt, 'enqueuedAt')
|
||||
]
|
||||
|
|
@ -703,7 +735,7 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
|
|||
`refs and nil are not allowed here`
|
||||
)
|
||||
}
|
||||
return [T.EMBED_LANDED, V, uuidToBytes(record.id), record.vector]
|
||||
return [T.EMBED_LANDED, V, C, K, uuidToBytes(record.id), record.vector]
|
||||
}
|
||||
case 'blob.manifest': {
|
||||
if (typeof record.mimeType !== 'string') {
|
||||
|
|
@ -715,6 +747,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
|
|||
return [
|
||||
T.BLOB_MANIFEST,
|
||||
V,
|
||||
C,
|
||||
K,
|
||||
hashToBytes(record.hash),
|
||||
toWireU64(record.size, 'blob size'),
|
||||
record.mimeType,
|
||||
|
|
@ -725,7 +759,7 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
|
|||
if (!isPlainMap(record.note)) {
|
||||
throw new Error('fact log v2: projection.note requires a map')
|
||||
}
|
||||
return [T.PROJECTION_NOTE, V, record.note]
|
||||
return [T.PROJECTION_NOTE, V, C, K, record.note]
|
||||
case 'bootstrap.baseline': {
|
||||
if (record.kind !== 'noun' && record.kind !== 'verb') {
|
||||
throw new Error(`fact log v2: bootstrap.baseline kind must be 'noun' or 'verb'`)
|
||||
|
|
@ -733,6 +767,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
|
|||
return [
|
||||
T.BOOTSTRAP_BASELINE,
|
||||
V,
|
||||
C,
|
||||
K,
|
||||
uuidToBytes(record.id),
|
||||
record.kind === 'noun' ? 0 : 1,
|
||||
record.metadata ?? null,
|
||||
|
|
@ -748,6 +784,8 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
|
|||
return [
|
||||
T.LOG_GENESIS,
|
||||
V,
|
||||
C,
|
||||
K,
|
||||
record.idSpaceWidth,
|
||||
uuidToBytes(record.brainId),
|
||||
toWireU64(record.createdAt, 'createdAt')
|
||||
|
|
@ -763,35 +801,39 @@ function encodeRecord(record: LogRecord, options: EncodeFactV2Options | undefine
|
|||
}
|
||||
}
|
||||
|
||||
/** Exact wire arity per record type (envelope of 2 + type-specific fields). */
|
||||
/** Exact wire arity per record type (envelope of 4 + type-specific fields). */
|
||||
const RECORD_ARITY: Record<number, number> = {
|
||||
[LOG_RECORD_TYPES.NOUN_AFTER_IMAGE]: 6,
|
||||
[LOG_RECORD_TYPES.NOUN_TOMBSTONE]: 3,
|
||||
[LOG_RECORD_TYPES.VERB_AFTER_IMAGE]: 11,
|
||||
[LOG_RECORD_TYPES.VERB_TOMBSTONE]: 3,
|
||||
[LOG_RECORD_TYPES.BATCH_META]: 3,
|
||||
[LOG_RECORD_TYPES.EMBED_PENDING]: 4,
|
||||
[LOG_RECORD_TYPES.EMBED_LANDED]: 4,
|
||||
[LOG_RECORD_TYPES.BLOB_MANIFEST]: 6,
|
||||
[LOG_RECORD_TYPES.PROJECTION_NOTE]: 3,
|
||||
[LOG_RECORD_TYPES.BOOTSTRAP_BASELINE]: 6,
|
||||
[LOG_RECORD_TYPES.LOG_GENESIS]: 5
|
||||
[LOG_RECORD_TYPES.NOUN_AFTER_IMAGE]: 8,
|
||||
[LOG_RECORD_TYPES.NOUN_TOMBSTONE]: 5,
|
||||
[LOG_RECORD_TYPES.VERB_AFTER_IMAGE]: 13,
|
||||
[LOG_RECORD_TYPES.VERB_TOMBSTONE]: 5,
|
||||
[LOG_RECORD_TYPES.BATCH_META]: 5,
|
||||
[LOG_RECORD_TYPES.EMBED_PENDING]: 6,
|
||||
[LOG_RECORD_TYPES.EMBED_LANDED]: 6,
|
||||
[LOG_RECORD_TYPES.BLOB_MANIFEST]: 8,
|
||||
[LOG_RECORD_TYPES.PROJECTION_NOTE]: 5,
|
||||
[LOG_RECORD_TYPES.BOOTSTRAP_BASELINE]: 8,
|
||||
[LOG_RECORD_TYPES.LOG_GENESIS]: 7
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode one wire record. Returns `null` for pads (skipped by definition).
|
||||
* Unknown type / newer version throw {@link UnknownLogRecordError} — never
|
||||
* skip-and-continue.
|
||||
* skip-and-continue. The reserved crypto envelope is verified BEFORE the
|
||||
* arity check (an encrypted record's field layout is a newer reader's
|
||||
* business, not a malformed-record error): any nonzero cipherFlag or non-nil
|
||||
* keyId refuses with the same typed error class.
|
||||
*/
|
||||
function decodeRecord(raw: unknown): LogRecord | null {
|
||||
if (!Array.isArray(raw) || raw.length < 2) {
|
||||
throw new Error('fact log v2: malformed record envelope (need [type, version, ...])')
|
||||
throw new Error('fact log v2: malformed record envelope (need [type, version, cipherFlag, keyId, ...])')
|
||||
}
|
||||
const recordType = wireToU8(raw[0], 'recordType')
|
||||
const recordVersion = wireToU8(raw[1], 'recordVersion')
|
||||
|
||||
if (recordType === LOG_RECORD_TYPES.PAD) {
|
||||
// Length-only filler: skipped wholesale, filler fields never inspected.
|
||||
// Length-only filler: skipped wholesale, filler fields never inspected
|
||||
// (pads therefore carry no crypto envelope — by definition, not omission).
|
||||
return null
|
||||
}
|
||||
const arity = RECORD_ARITY[recordType]
|
||||
|
|
@ -814,6 +856,20 @@ function decodeRecord(raw: unknown): LogRecord | null {
|
|||
if (recordVersion !== LOG_RECORD_VERSION) {
|
||||
throw new Error(`fact log v2: record type ${recordType} has invalid record version ${recordVersion}`)
|
||||
}
|
||||
if (raw.length < 4) {
|
||||
throw new Error('fact log v2: malformed record envelope (need [type, version, cipherFlag, keyId, ...])')
|
||||
}
|
||||
const cipherFlag = wireToU8(raw[2], 'cipherFlag')
|
||||
const keyId = raw[3]
|
||||
if (cipherFlag !== LOG_RECORD_CIPHER_PLAINTEXT || (keyId !== null && keyId !== undefined)) {
|
||||
throw new UnknownLogRecordError(
|
||||
recordType,
|
||||
recordVersion,
|
||||
`fact log v2: record type ${recordType} carries cipherFlag ${cipherFlag}` +
|
||||
`${keyId !== null && keyId !== undefined ? ' and a keyId' : ''} — ` +
|
||||
`encrypted records need a newer reader`
|
||||
)
|
||||
}
|
||||
if (raw.length !== arity) {
|
||||
throw new Error(
|
||||
`fact log v2: record type ${recordType} expects ${arity} wire fields; got ${raw.length}`
|
||||
|
|
@ -824,94 +880,94 @@ function decodeRecord(raw: unknown): LogRecord | null {
|
|||
case LOG_RECORD_TYPES.NOUN_AFTER_IMAGE:
|
||||
return {
|
||||
type: 'noun.afterImage',
|
||||
id: bytesToUuid(raw[2], 'noun.afterImage id'),
|
||||
entityInt: wireToBigint(raw[3], 'entityInt'),
|
||||
metadata: raw[4] ?? null,
|
||||
vectorLeg: decodeVectorLeg(raw[5], 'noun.afterImage')
|
||||
id: bytesToUuid(raw[4], 'noun.afterImage id'),
|
||||
entityInt: wireToBigint(raw[5], 'entityInt'),
|
||||
metadata: raw[6] ?? null,
|
||||
vectorLeg: decodeVectorLeg(raw[7], 'noun.afterImage')
|
||||
}
|
||||
case LOG_RECORD_TYPES.NOUN_TOMBSTONE:
|
||||
return { type: 'noun.tombstone', id: bytesToUuid(raw[2], 'noun.tombstone id') }
|
||||
return { type: 'noun.tombstone', id: bytesToUuid(raw[4], 'noun.tombstone id') }
|
||||
case LOG_RECORD_TYPES.VERB_AFTER_IMAGE: {
|
||||
if (typeof raw[6] !== 'string') {
|
||||
if (typeof raw[8] !== 'string') {
|
||||
throw new Error('fact log v2: verb.afterImage verb name is not a string')
|
||||
}
|
||||
return {
|
||||
type: 'verb.afterImage',
|
||||
id: bytesToUuid(raw[2], 'verb.afterImage id'),
|
||||
verbInt: wireToBigint(raw[3], 'verbInt'),
|
||||
metadata: raw[4] ?? null,
|
||||
vectorLeg: decodeVectorLeg(raw[5], 'verb.afterImage'),
|
||||
verb: raw[6],
|
||||
sourceId: bytesToUuid(raw[7], 'verb.afterImage sourceId'),
|
||||
sourceInt: wireToBigint(raw[8], 'sourceInt'),
|
||||
targetId: bytesToUuid(raw[9], 'verb.afterImage targetId'),
|
||||
targetInt: wireToBigint(raw[10], 'targetInt')
|
||||
id: bytesToUuid(raw[4], 'verb.afterImage id'),
|
||||
verbInt: wireToBigint(raw[5], 'verbInt'),
|
||||
metadata: raw[6] ?? null,
|
||||
vectorLeg: decodeVectorLeg(raw[7], 'verb.afterImage'),
|
||||
verb: raw[8],
|
||||
sourceId: bytesToUuid(raw[9], 'verb.afterImage sourceId'),
|
||||
sourceInt: wireToBigint(raw[10], 'sourceInt'),
|
||||
targetId: bytesToUuid(raw[11], 'verb.afterImage targetId'),
|
||||
targetInt: wireToBigint(raw[12], 'targetInt')
|
||||
}
|
||||
}
|
||||
case LOG_RECORD_TYPES.VERB_TOMBSTONE:
|
||||
return { type: 'verb.tombstone', id: bytesToUuid(raw[2], 'verb.tombstone id') }
|
||||
return { type: 'verb.tombstone', id: bytesToUuid(raw[4], 'verb.tombstone id') }
|
||||
case LOG_RECORD_TYPES.BATCH_META: {
|
||||
if (!isPlainMap(raw[2])) throw new Error('fact log v2: batch.meta payload is not a map')
|
||||
return { type: 'batch.meta', meta: raw[2] }
|
||||
if (!isPlainMap(raw[4])) throw new Error('fact log v2: batch.meta payload is not a map')
|
||||
return { type: 'batch.meta', meta: raw[4] }
|
||||
}
|
||||
case LOG_RECORD_TYPES.EMBED_PENDING:
|
||||
return {
|
||||
type: 'embed.pending',
|
||||
id: bytesToUuid(raw[2], 'embed.pending id'),
|
||||
enqueuedAt: wireToNumber(raw[3], 'enqueuedAt')
|
||||
id: bytesToUuid(raw[4], 'embed.pending id'),
|
||||
enqueuedAt: wireToNumber(raw[5], 'enqueuedAt')
|
||||
}
|
||||
case LOG_RECORD_TYPES.EMBED_LANDED: {
|
||||
const leg = decodeVectorLeg(raw[3], 'embed.landed')
|
||||
const leg = decodeVectorLeg(raw[5], 'embed.landed')
|
||||
if (!Array.isArray(leg)) {
|
||||
throw new Error(
|
||||
'fact log v2: embed.landed must carry an INLINE float vector — refs and nil are not allowed here'
|
||||
)
|
||||
}
|
||||
return { type: 'embed.landed', id: bytesToUuid(raw[2], 'embed.landed id'), vector: leg }
|
||||
return { type: 'embed.landed', id: bytesToUuid(raw[4], 'embed.landed id'), vector: leg }
|
||||
}
|
||||
case LOG_RECORD_TYPES.BLOB_MANIFEST: {
|
||||
if (typeof raw[4] !== 'string') {
|
||||
if (typeof raw[6] !== 'string') {
|
||||
throw new Error('fact log v2: blob.manifest mimeType is not a string')
|
||||
}
|
||||
const refOp = wireToU8(raw[5], 'refOp')
|
||||
const refOp = wireToU8(raw[7], 'refOp')
|
||||
if (refOp !== 0 && refOp !== 1) {
|
||||
throw new Error(`fact log v2: blob.manifest refOp must be 0 (add) or 1 (release); got ${refOp}`)
|
||||
}
|
||||
return {
|
||||
type: 'blob.manifest',
|
||||
hash: bytesToHash(raw[2]),
|
||||
size: wireToNumber(raw[3], 'blob size'),
|
||||
mimeType: raw[4],
|
||||
hash: bytesToHash(raw[4]),
|
||||
size: wireToNumber(raw[5], 'blob size'),
|
||||
mimeType: raw[6],
|
||||
refOp: refOp === 0 ? 'add' : 'release'
|
||||
}
|
||||
}
|
||||
case LOG_RECORD_TYPES.PROJECTION_NOTE: {
|
||||
if (!isPlainMap(raw[2])) throw new Error('fact log v2: projection.note payload is not a map')
|
||||
return { type: 'projection.note', note: raw[2] }
|
||||
if (!isPlainMap(raw[4])) throw new Error('fact log v2: projection.note payload is not a map')
|
||||
return { type: 'projection.note', note: raw[4] }
|
||||
}
|
||||
case LOG_RECORD_TYPES.BOOTSTRAP_BASELINE: {
|
||||
const kind = wireToU8(raw[3], 'bootstrap.baseline kind')
|
||||
const kind = wireToU8(raw[5], 'bootstrap.baseline kind')
|
||||
if (kind !== 0 && kind !== 1) {
|
||||
throw new Error(`fact log v2: bootstrap.baseline kind must be 0 (noun) or 1 (verb); got ${kind}`)
|
||||
}
|
||||
return {
|
||||
type: 'bootstrap.baseline',
|
||||
id: bytesToUuid(raw[2], 'bootstrap.baseline id'),
|
||||
id: bytesToUuid(raw[4], 'bootstrap.baseline id'),
|
||||
kind: kind === 0 ? 'noun' : 'verb',
|
||||
metadata: raw[4] ?? null,
|
||||
vectorLeg: decodeVectorLeg(raw[5], 'bootstrap.baseline')
|
||||
metadata: raw[6] ?? null,
|
||||
vectorLeg: decodeVectorLeg(raw[7], 'bootstrap.baseline')
|
||||
}
|
||||
}
|
||||
case LOG_RECORD_TYPES.LOG_GENESIS: {
|
||||
const width = wireToU8(raw[2], 'idSpaceWidth')
|
||||
const width = wireToU8(raw[4], 'idSpaceWidth')
|
||||
if (width !== 32 && width !== 64) {
|
||||
throw new Error(`fact log v2: log.genesis idSpaceWidth must be 32 or 64; got ${width}`)
|
||||
}
|
||||
return {
|
||||
type: 'log.genesis',
|
||||
idSpaceWidth: width,
|
||||
brainId: bytesToUuid(raw[3], 'log.genesis brainId'),
|
||||
createdAt: wireToNumber(raw[4], 'createdAt')
|
||||
brainId: bytesToUuid(raw[5], 'log.genesis brainId'),
|
||||
createdAt: wireToNumber(raw[6], 'createdAt')
|
||||
}
|
||||
}
|
||||
default:
|
||||
|
|
@ -944,8 +1000,12 @@ export function encodeFactV2(fact: CommitFactV2, options?: EncodeFactV2Options):
|
|||
if (!Number.isSafeInteger(fact.timestamp) || fact.timestamp < 0) {
|
||||
throw new Error(`fact log v2: timestamp must be a non-negative integer; got ${fact.timestamp}`)
|
||||
}
|
||||
if (!Array.isArray(fact.records) || fact.records.length === 0) {
|
||||
throw new Error('fact log v2: a fact must carry at least one record')
|
||||
// records MAY be empty: a committed generation whose ops all collapsed
|
||||
// (e.g. a batch whose relates deduped to no-ops) is still a real
|
||||
// generation — v1 encoded empty ops the same way; refusing here would
|
||||
// fork the two formats' commit semantics.
|
||||
if (!Array.isArray(fact.records)) {
|
||||
throw new Error('fact log v2: records must be an array')
|
||||
}
|
||||
if (fact.meta !== undefined && !isPlainMap(fact.meta)) {
|
||||
throw new Error('fact log v2: fact meta must be a map when present')
|
||||
|
|
@ -1092,9 +1152,15 @@ function decodeFactV2(payload: Uint8Array, options?: DecodeFactV2Options): Commi
|
|||
// Sector seals
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Smallest constructible pad frame (envelope + bare pad record), memoized. */
|
||||
/**
|
||||
* Smallest constructible pad frame in bytes (frame prefix + the bare pad
|
||||
* record fact), memoized. Exported for streaming writers that pad an
|
||||
* append-only tail to a seal boundary: a gap smaller than this cannot hold
|
||||
* any frame, so the writer pads through one extra sector (the same rule
|
||||
* {@link sealGroup} applies).
|
||||
*/
|
||||
let minPadFrameBytesMemo: number | null = null
|
||||
function minPadFrameBytes(): number {
|
||||
export function minPadFrameBytes(): number {
|
||||
if (minPadFrameBytesMemo === null) {
|
||||
minPadFrameBytesMemo =
|
||||
FRAME_PREFIX_BYTES +
|
||||
|
|
@ -1145,6 +1211,25 @@ function buildPadFrame(totalBytes: number): Uint8Array {
|
|||
return buildFrame(payload)
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a pad frame of EXACTLY `totalBytes` — the streaming-append counterpart
|
||||
* of {@link sealGroup} for writers that append pads directly to a live tail
|
||||
* instead of sealing an in-memory group. Refuses sizes smaller than the
|
||||
* smallest constructible pad frame ({@link minPadFrameBytes}); readers skip
|
||||
* the result by definition (a type-0 record is length-only filler).
|
||||
*
|
||||
* @param totalBytes - The exact frame size to construct (prefix included).
|
||||
* @returns The complete pad frame bytes.
|
||||
*/
|
||||
export function encodePadFrame(totalBytes: number): Uint8Array {
|
||||
if (!Number.isInteger(totalBytes) || totalBytes < minPadFrameBytes()) {
|
||||
throw new Error(
|
||||
`fact log v2: a pad frame must be at least ${minPadFrameBytes()} bytes; got ${totalBytes}`
|
||||
)
|
||||
}
|
||||
return buildPadFrame(totalBytes)
|
||||
}
|
||||
|
||||
/**
|
||||
* Seal a group of frames to a sector boundary: concatenate the frames and pad
|
||||
* to the next `sealSize` multiple with ONE pad frame. An already-aligned
|
||||
|
|
|
|||
|
|
@ -46,7 +46,13 @@ import type {
|
|||
TxLogEntry
|
||||
} from './types.js'
|
||||
import { readLogAuthority } from './logAuthority.js'
|
||||
import { FactLog, storageSupportsFactLog, type CommitFact, type FactOp } from './factLog.js'
|
||||
import {
|
||||
FactLog,
|
||||
storageSupportsFactLog,
|
||||
type CommitFact,
|
||||
type FactOp,
|
||||
type FactIntMinter
|
||||
} from './factLog.js'
|
||||
import { GenerationSegmentStore, type FoldGeneration } from './generationSegments.js'
|
||||
import { crc32c } from '../utils/crc32c.js'
|
||||
|
||||
|
|
@ -182,6 +188,22 @@ export class GenerationStore {
|
|||
this.logDurability = mode
|
||||
}
|
||||
|
||||
/**
|
||||
* The fact log's v2 int minter — injected by the OWNER (brainy wires the
|
||||
* metadata index's id mapper here right after the index is ready), because
|
||||
* this store cannot know the mapper. With the minter installed, new fact
|
||||
* segments write the v2 format and after-image records carry minted dense
|
||||
* ints reproducible by an id-mapper rebuild. Survives reopen: `open()`
|
||||
* re-installs it on the fresh {@link FactLog} instance.
|
||||
*/
|
||||
private intMinter: FactIntMinter | null = null
|
||||
|
||||
/** Install the fact log's v2 int minter (see {@link intMinter}). */
|
||||
setIntMinter(mint: FactIntMinter): void {
|
||||
this.intMinter = mint
|
||||
this.factLog?.setIntMinter(mint)
|
||||
}
|
||||
|
||||
/** Latest reserved/observed generation (≥ {@link committed}). */
|
||||
private counter = 0
|
||||
/** Committed-transaction watermark (manifest generation). */
|
||||
|
|
@ -493,6 +515,7 @@ export class GenerationStore {
|
|||
// hosts no fact log (readers fall back to canonical enumeration).
|
||||
if (storageSupportsFactLog(this.storage)) {
|
||||
this.factLog = new FactLog(this.storage)
|
||||
if (this.intMinter) this.factLog.setIntMinter(this.intMinter)
|
||||
// LOG-AUTHORITY REPLAY (durable-at-ack's recovery half): when this
|
||||
// brain's stored authority is the log, an intact fact ABOVE the
|
||||
// manifest is an ACKED write whose canonical bytes may not have
|
||||
|
|
|
|||
389
tests/integration/fact-log-v2-cutover.test.ts
Normal file
389
tests/integration/fact-log-v2-cutover.test.ts
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
/**
|
||||
* @module tests/integration/fact-log-v2-cutover
|
||||
* @description The fact log's LIVE WRITE FORMAT cutover to v2, end-to-end
|
||||
* through real brains: (a) a NEW brain's tail segment carries a v2 header
|
||||
* (formatVersion 2, sealSize 4096), opens with the log.genesis record
|
||||
* (id-space width 64 + the manifest-persisted brainId), and scanFacts yields
|
||||
* the same CommitFact shape a v1 brain would — reconstruction included,
|
||||
* proven by digest-equality against canonical after a reopen; (b) MIXED
|
||||
* logs: an existing v1 segment stays readable forever beside a v2 tail
|
||||
* (cutover-by-rotation; the v1 segment is never rewritten); (c) MINT:
|
||||
* after-image records carry the metadata index id mapper's exact int
|
||||
* assignments (white-box compare); (d) SEALS: every flush leaves the tail
|
||||
* sector-aligned, and pads are invisible to scans; (e) REPLAY: the
|
||||
* log-authority recovery path resurrects an acked write from a v2 tail
|
||||
* after a crash-style abandon.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
import {
|
||||
parseSegmentHeader,
|
||||
decodeGroupV2,
|
||||
SEGMENT_HEADER_BYTES,
|
||||
FACT_LOG_FORMAT_V1,
|
||||
FACT_LOG_FORMAT_V2,
|
||||
type LogGenesisRecord,
|
||||
type NounAfterImageRecord
|
||||
} from '../../src/db/factLogFormat.js'
|
||||
import type { CommitFact, FactIntMinter, FactLog } from '../../src/db/factLog.js'
|
||||
import {
|
||||
makeTempDir,
|
||||
openBrain,
|
||||
storeOf,
|
||||
abandonAsCrashed,
|
||||
factGenerations,
|
||||
vec,
|
||||
uid
|
||||
} from '../helpers/durabilityKillMatrix.js'
|
||||
|
||||
/** The VFS root — created at init by a baseline (generation-less) write. */
|
||||
const VFS_ROOT = '00000000-0000-0000-0000-000000000000'
|
||||
const FACTS_DIR = ['_generations', 'facts'] as const
|
||||
const MANIFEST_PATH = '_generations/facts/manifest.json'
|
||||
|
||||
/** White-box internals this suite instruments. */
|
||||
type BrainInternals = {
|
||||
storage: {
|
||||
readRawObject(p: string): Promise<unknown | null>
|
||||
readNounRaw(id: string): Promise<{ metadata: unknown | null; vector: unknown | null }>
|
||||
}
|
||||
metadataIndex: {
|
||||
getIdMapper(): { getInt(uuid: string): number | undefined }
|
||||
}
|
||||
}
|
||||
const internals = (brain: Brainy): BrainInternals => brain as unknown as BrainInternals
|
||||
|
||||
/** The facts manifest as stored (additive brainId included). */
|
||||
interface StoredFactsManifest {
|
||||
segments: Array<{ file: string }>
|
||||
tailSegment: string | null
|
||||
brainId?: string
|
||||
}
|
||||
|
||||
async function readManifest(brain: Brainy): Promise<StoredFactsManifest> {
|
||||
const manifest = (await internals(brain).storage.readRawObject(
|
||||
MANIFEST_PATH
|
||||
)) as StoredFactsManifest | null
|
||||
expect(manifest, 'the facts manifest exists').toBeTruthy()
|
||||
return manifest!
|
||||
}
|
||||
|
||||
/** Raw on-disk bytes of one fact segment file. */
|
||||
function segmentBytes(dir: string, file: string): Uint8Array {
|
||||
return new Uint8Array(fs.readFileSync(path.join(dir, ...FACTS_DIR, file)))
|
||||
}
|
||||
|
||||
async function allFacts(brain: Brainy): Promise<CommitFact[]> {
|
||||
const scan = (brain as unknown as { scanFacts(): { batches(): AsyncGenerator<{ facts: CommitFact[] }> } | null }).scanFacts()
|
||||
expect(scan, 'this storage hosts a fact log').not.toBeNull()
|
||||
const facts: CommitFact[] = []
|
||||
for await (const batch of scan!.batches()) facts.push(...batch.facts)
|
||||
return facts
|
||||
}
|
||||
|
||||
/** The live FactLog instance (white-box: the minter strip in scenario b). */
|
||||
function factLogOf(brain: Brainy): FactLog & { intMinter: FactIntMinter | null } {
|
||||
const log = storeOf(brain).getFactLog()
|
||||
expect(log, 'filesystem storage hosts a fact log').not.toBeNull()
|
||||
return log as FactLog & { intMinter: FactIntMinter | null }
|
||||
}
|
||||
|
||||
describe('fact log v2 cutover — live writes land in the v2 segment format', () => {
|
||||
const dirs: string[] = []
|
||||
const brains: Brainy[] = []
|
||||
|
||||
const trackDir = (): string => {
|
||||
const dir = makeTempDir()
|
||||
dirs.push(dir)
|
||||
return dir
|
||||
}
|
||||
const track = (brain: Brainy): Brainy => {
|
||||
brains.push(brain)
|
||||
return brain
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) {
|
||||
await (b as unknown as { close?: () => Promise<void> }).close?.().catch(() => {})
|
||||
}
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('(a) NEW BRAIN: v2 tail header, genesis-first, and scanFacts parity with canonical across a reopen', async () => {
|
||||
const dir = trackDir()
|
||||
const brain = track(await openBrain(dir))
|
||||
const idA = uid('v2-new-a')
|
||||
const idB = uid('v2-new-b')
|
||||
await brain.add({ id: idA, data: 'alpha', type: NounType.Document, vector: vec(1), metadata: { n: 1 } })
|
||||
await brain.add({ id: idB, data: 'beta', type: NounType.Document, vector: vec(2), metadata: { n: 2 } })
|
||||
await brain.flush()
|
||||
|
||||
// The tail segment's raw header bytes: formatVersion 2, sealSize 4096.
|
||||
const manifest = await readManifest(brain)
|
||||
expect(manifest.tailSegment).toBeTruthy()
|
||||
expect(manifest.brainId, 'the brain id was minted into the manifest').toBeTruthy()
|
||||
const bytes = segmentBytes(dir, manifest.tailSegment!)
|
||||
const header = parseSegmentHeader(bytes.subarray(0, SEGMENT_HEADER_BYTES))
|
||||
expect(header.formatVersion).toBe(FACT_LOG_FORMAT_V2)
|
||||
expect(header.sealSize).toBe(4096)
|
||||
|
||||
// Genesis is the FIRST record of the FIRST fact — and appears exactly once.
|
||||
const group = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES), { expectedIdSpaceWidth: 64 })
|
||||
expect(group.facts.length).toBeGreaterThanOrEqual(2)
|
||||
const firstRecord = group.facts[0].records[0]
|
||||
expect(firstRecord.type).toBe('log.genesis')
|
||||
const genesis = firstRecord as LogGenesisRecord
|
||||
expect(genesis.idSpaceWidth).toBe(64)
|
||||
expect(genesis.brainId).toBe(manifest.brainId)
|
||||
const genesisCount = group.facts
|
||||
.flatMap((f) => f.records)
|
||||
.filter((r) => r.type === 'log.genesis').length
|
||||
expect(genesisCount).toBe(1)
|
||||
|
||||
// Shape parity + reconstruction fidelity: REOPEN (so the tail decodes
|
||||
// from disk, not from the in-session originals) and compare each add's
|
||||
// CommitFact op against canonical byte truth — metadata leg (bigint
|
||||
// timestamps normalized back to numbers) AND the reconstructed vector
|
||||
// wrapper must equal what readNounRaw returns, exactly as a v1 log's
|
||||
// byte-faithful capture would.
|
||||
await (brain as unknown as { close: () => Promise<void> }).close()
|
||||
brains.splice(brains.indexOf(brain), 1)
|
||||
const reopened = track(await openBrain(dir))
|
||||
const facts = await allFacts(reopened)
|
||||
const gens = facts.map((f) => f.generation)
|
||||
expect([...gens].sort((a, b) => a - b)).toEqual(gens)
|
||||
expect(new Set(gens).size).toBe(gens.length)
|
||||
|
||||
const logGens = new Set(
|
||||
((await (reopened as unknown as { transactionLog(): Promise<Array<{ generation: number }>> }).transactionLog()) ?? []).map(
|
||||
(e) => e.generation
|
||||
)
|
||||
)
|
||||
for (const g of gens) expect(logGens.has(g), `generation ${g} is a real commit`).toBe(true)
|
||||
|
||||
for (const id of [idA, idB]) {
|
||||
const fact = facts.find((f) => f.ops.some((op) => op.id === id && op.record !== null))
|
||||
expect(fact, `the add fact for ${id} survives the reopen`).toBeDefined()
|
||||
const op = fact!.ops.find((o) => o.id === id)!
|
||||
expect(op.kind).toBe('noun')
|
||||
const canonical = await internals(reopened).storage.readNounRaw(id)
|
||||
expect(op.record!.metadata).toStrictEqual(canonical.metadata)
|
||||
expect(op.record!.vector).toStrictEqual(canonical.vector)
|
||||
}
|
||||
})
|
||||
|
||||
it('(b) MIXED LOG: an existing v1 segment stays readable forever beside the v2 tail (cutover by rotation, v1 bytes untouched)', async () => {
|
||||
// ROUTE: a REAL v1 segment is written by the v1 writer itself — the live
|
||||
// FactLog with its minter stripped (the exact pre-cutover code path,
|
||||
// still shipped for minter-less configurations) — then the minter is
|
||||
// restored mid-session and the next append performs the cutover
|
||||
// rotation. Stronger than hand-crafted bytes: both formats come from
|
||||
// their real writers, on one log.
|
||||
const dir = trackDir()
|
||||
const brain = track(await openBrain(dir))
|
||||
const log = factLogOf(brain)
|
||||
const minter = log.intMinter
|
||||
expect(minter, 'the brain wired the int minter at init').toBeTruthy()
|
||||
|
||||
log.intMinter = null // the pre-cutover writer
|
||||
const idOld1 = uid('v1-old-1')
|
||||
const idOld2 = uid('v1-old-2')
|
||||
await brain.add({ id: idOld1, data: 'old one', type: NounType.Document, vector: vec(3), metadata: { era: 'v1' } })
|
||||
await brain.add({ id: idOld2, data: 'old two', type: NounType.Document, vector: vec(4), metadata: { era: 'v1' } })
|
||||
await brain.flush()
|
||||
|
||||
const before = await readManifest(brain)
|
||||
expect(before.segments).toHaveLength(0)
|
||||
const v1TailFile = before.tailSegment!
|
||||
const v1Bytes = segmentBytes(dir, v1TailFile)
|
||||
expect(parseSegmentHeader(v1Bytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe(
|
||||
FACT_LOG_FORMAT_V1
|
||||
)
|
||||
|
||||
log.intMinter = minter // the cutover lands mid-session
|
||||
const idNew = uid('v2-new')
|
||||
await brain.add({ id: idNew, data: 'new era', type: NounType.Document, vector: vec(5), metadata: { era: 'v2' } })
|
||||
await brain.flush()
|
||||
|
||||
// The v1 tail was SEALED (bytes untouched), the new tail is v2.
|
||||
const after = await readManifest(brain)
|
||||
expect(after.segments.map((s) => s.file)).toContain(v1TailFile)
|
||||
expect(after.tailSegment).not.toBe(v1TailFile)
|
||||
const sealedBytes = segmentBytes(dir, v1TailFile)
|
||||
expect(parseSegmentHeader(sealedBytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe(
|
||||
FACT_LOG_FORMAT_V1
|
||||
)
|
||||
expect(
|
||||
Buffer.compare(Buffer.from(sealedBytes), Buffer.from(v1Bytes)),
|
||||
'the sealed v1 segment is byte-identical — never rewritten'
|
||||
).toBe(0)
|
||||
const tailBytes = segmentBytes(dir, after.tailSegment!)
|
||||
expect(parseSegmentHeader(tailBytes.subarray(0, SEGMENT_HEADER_BYTES)).formatVersion).toBe(
|
||||
FACT_LOG_FORMAT_V2
|
||||
)
|
||||
// NOT a brand-new log: no genesis on a rotated-in v2 tail.
|
||||
const tailGroup = decodeGroupV2(tailBytes.subarray(SEGMENT_HEADER_BYTES), {
|
||||
expectedIdSpaceWidth: 64
|
||||
})
|
||||
expect(
|
||||
tailGroup.facts.flatMap((f) => f.records).some((r) => r.type === 'log.genesis')
|
||||
).toBe(false)
|
||||
|
||||
// One scan spans both formats, shape-identically, in generation order.
|
||||
const liveFacts = await allFacts(brain)
|
||||
const liveGens = liveFacts.map((f) => f.generation)
|
||||
expect([...liveGens].sort((a, b) => a - b)).toEqual(liveGens)
|
||||
for (const id of [idOld1, idOld2, idNew]) {
|
||||
const fact = liveFacts.find((f) => f.ops.some((op) => op.id === id))
|
||||
expect(fact, `fact for ${id} is scannable`).toBeDefined()
|
||||
const op = fact!.ops.find((o) => o.id === id)!
|
||||
expect(op.kind).toBe('noun')
|
||||
expect(op.record).not.toBeNull()
|
||||
}
|
||||
|
||||
// The MIXED log survives a reopen and keeps appending (v2 tail).
|
||||
await (brain as unknown as { close: () => Promise<void> }).close()
|
||||
brains.splice(brains.indexOf(brain), 1)
|
||||
const reopened = track(await openBrain(dir))
|
||||
const reFacts = await allFacts(reopened)
|
||||
expect(reFacts.map((f) => f.generation)).toEqual(liveGens)
|
||||
// The v1 fact still reads exactly as the v1 decoder always read it.
|
||||
// (Not compared byte-strict against canonical: the v1 CAPTURE has a
|
||||
// known pre-existing wart — write-cache-warm objects carry
|
||||
// undefined-valued engine keys that msgpack preserves as nil while the
|
||||
// durable JSON drops them. v1 bytes are frozen; the v2 encoder
|
||||
// sanitizes to durable truth instead — pinned in scenario (a).)
|
||||
const oldOp = reFacts
|
||||
.find((f) => f.ops.some((op) => op.id === idOld1))!
|
||||
.ops.find((o) => o.id === idOld1)!
|
||||
const canonicalOld = await internals(reopened).storage.readNounRaw(idOld1)
|
||||
const oldMeta = oldOp.record!.metadata as Record<string, unknown>
|
||||
expect(oldMeta.noun).toBe('document')
|
||||
expect((oldMeta.metadata as Record<string, unknown>).era).toBe('v1')
|
||||
const oldWrapper = oldOp.record!.vector as { id: string; vector: number[] }
|
||||
const canonicalWrapper = canonicalOld.vector as { id: string; vector: number[] }
|
||||
expect(oldWrapper.id).toBe(idOld1)
|
||||
expect(oldWrapper.vector).toStrictEqual(canonicalWrapper.vector)
|
||||
await reopened.add({ id: uid('post-reopen'), data: 'still writing', type: NounType.Document, vector: vec(6), metadata: {} })
|
||||
expect((await factGenerations(reopened)).length).toBe(liveGens.length + 1)
|
||||
})
|
||||
|
||||
it('(c) MINT-AT-APPEND: after-image records carry the id mapper\'s EXACT int assignments — distinct, nonzero, reproducible', async () => {
|
||||
const dir = trackDir()
|
||||
const brain = track(await openBrain(dir))
|
||||
const idA = uid('mint-a')
|
||||
const idB = uid('mint-b')
|
||||
await brain.add({ id: idA, data: 'mint one', type: NounType.Document, vector: vec(7), metadata: { m: 1 } })
|
||||
await brain.add({ id: idB, data: 'mint two', type: NounType.Document, vector: vec(8), metadata: { m: 2 } })
|
||||
await brain.flush()
|
||||
|
||||
const manifest = await readManifest(brain)
|
||||
const bytes = segmentBytes(dir, manifest.tailSegment!)
|
||||
const group = decodeGroupV2(bytes.subarray(SEGMENT_HEADER_BYTES), { expectedIdSpaceWidth: 64 })
|
||||
const afterImages = new Map<string, NounAfterImageRecord>()
|
||||
for (const fact of group.facts) {
|
||||
for (const record of fact.records) {
|
||||
if (record.type === 'noun.afterImage') afterImages.set(record.id, record)
|
||||
}
|
||||
}
|
||||
const recA = afterImages.get(idA)
|
||||
const recB = afterImages.get(idB)
|
||||
expect(recA, 'idA has a decoded after-image').toBeDefined()
|
||||
expect(recB, 'idB has a decoded after-image').toBeDefined()
|
||||
expect(recA!.entityInt).toBeGreaterThan(0n)
|
||||
expect(recB!.entityInt).toBeGreaterThan(0n)
|
||||
expect(recA!.entityInt).not.toBe(recB!.entityInt)
|
||||
|
||||
// White-box: the ints on the wire ARE the metadata index mapper's
|
||||
// assignments — the exact ints a mapper rebuild must reproduce.
|
||||
const mapper = internals(brain).metadataIndex.getIdMapper()
|
||||
expect(recA!.entityInt).toBe(BigInt(mapper.getInt(idA)!))
|
||||
expect(recB!.entityInt).toBe(BigInt(mapper.getInt(idB)!))
|
||||
})
|
||||
|
||||
it('(d) SEALS AT SYNC: every flush leaves the tail sector-aligned; pads are invisible to scans', async () => {
|
||||
const dir = trackDir()
|
||||
const brain = track(await openBrain(dir))
|
||||
await brain.add({ id: uid('seal-1'), data: 'one', type: NounType.Document, vector: vec(10), metadata: {} })
|
||||
await brain.flush()
|
||||
|
||||
const manifest = await readManifest(brain)
|
||||
const tailPath = path.join(dir, ...FACTS_DIR, manifest.tailSegment!)
|
||||
const sizeAfterFirstFlush = fs.statSync(tailPath).size
|
||||
expect(sizeAfterFirstFlush).toBeGreaterThan(0)
|
||||
expect(sizeAfterFirstFlush % 4096, 'tail is sector-aligned after flush').toBe(0)
|
||||
const countAfterFirstFlush = (await factGenerations(brain)).length
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await brain.add({ id: uid(`seal-more-${i}`), data: `more ${i}`, type: NounType.Document, vector: vec(11 + i), metadata: { i } })
|
||||
}
|
||||
await brain.flush()
|
||||
const sizeAfterSecondFlush = fs.statSync(tailPath).size
|
||||
expect(sizeAfterSecondFlush).toBeGreaterThan(sizeAfterFirstFlush)
|
||||
expect(sizeAfterSecondFlush % 4096, 'still aligned after more writes + flush').toBe(0)
|
||||
|
||||
// Pads count toward bytes, never toward facts.
|
||||
expect((await factGenerations(brain)).length).toBe(countAfterFirstFlush + 3)
|
||||
})
|
||||
|
||||
it('(e) REPLAY COMPAT: the log-authority recovery path resurrects an acked write from a v2 tail after a crash-style abandon', async () => {
|
||||
// The flip idiom from the log-authority suite: seed writes, baseline
|
||||
// backfill LAST (the init-time VFS root never got a fact), flush, then
|
||||
// the sanctioned guarded flip — the oracle goes green over an ALL-V2
|
||||
// log, which is itself the reproduction proof for the v2 record path.
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-v2-cutover-'))
|
||||
dirs.push(dir)
|
||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||
const open = async (): Promise<Brainy> => {
|
||||
const b = new Brainy({
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
requireSubtype: false,
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
await b.init()
|
||||
return track(b)
|
||||
}
|
||||
|
||||
const brain = await open()
|
||||
const kept = await brain.add({ data: 'alpha document', type: 'document', metadata: { n: 1 } })
|
||||
const removed = await brain.add({ data: 'beta document', type: 'document', metadata: { n: 2 } })
|
||||
await brain.update({ id: kept, metadata: { n: 10 } })
|
||||
await brain.remove(removed)
|
||||
const root = await brain.get(VFS_ROOT)
|
||||
expect(root, 'the VFS root exists').toBeTruthy()
|
||||
await brain.update({ id: VFS_ROOT, metadata: root!.metadata }) // baseline backfill — final write
|
||||
await brain.flush()
|
||||
|
||||
const report = await (brain as unknown as { adoptLogAuthority(): Promise<{ verdict: string }> }).adoptLogAuthority()
|
||||
expect(report.verdict, 'the oracle is green over a pure-v2 log').toBe('green')
|
||||
|
||||
// An at-ack write: its v2 fact is fsynced (sector-sealed) at ack.
|
||||
const survivor = await brain.add({
|
||||
data: 'survives power loss',
|
||||
type: 'document',
|
||||
metadata: { s: 1 }
|
||||
})
|
||||
|
||||
// Crash-style abandon: RAM state gone, no flush, no close.
|
||||
await abandonAsCrashed(brain)
|
||||
|
||||
// Reopen: open() finds the acked fact ABOVE the manifest watermark in
|
||||
// the v2 tail (peekFactsAbove → v2 decode) and REPLAYS it into
|
||||
// canonical — an acked write is never lost.
|
||||
const reopened = await open()
|
||||
expect(
|
||||
(reopened as unknown as { logAuthority(): { authority: string } }).logAuthority().authority
|
||||
).toBe('log')
|
||||
const resurrected = await reopened.get(survivor)
|
||||
expect(resurrected, 'the acked write survived the crash').toBeTruthy()
|
||||
expect((resurrected as { metadata?: { s?: number } }).metadata?.s).toBe(1)
|
||||
expect((await factGenerations(reopened)).length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
|
@ -41,6 +41,7 @@ type BrainInternals = {
|
|||
saveNoun(n: unknown): Promise<void>
|
||||
saveNounMetadata(id: string, m: Record<string, unknown>): Promise<void>
|
||||
getNounMetadata(id: string): Promise<Record<string, unknown> | null>
|
||||
writeNounRaw(id: string, r: { metadata: null; vector: null }): Promise<void>
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -219,28 +220,21 @@ describe('log authority — the switch, the oracle, the guarded flip', () => {
|
|||
})
|
||||
})
|
||||
|
||||
it('THE FLIP REFUSES ON RED: names the oracle verdict and the cure, writes nothing, changes nothing', async () => {
|
||||
it('THE FLIP REFUSES ON A LOG-AHEAD DIVERGENCE: the witness denies what the log claims — nothing written, nothing changed', async () => {
|
||||
// Contract update (adoptLogAuthority's baseline backfill): curable
|
||||
// divergences — pre-log records and witness drift — are re-committed
|
||||
// and the flip proceeds; ONLY log-AHEAD divergences (the log claims
|
||||
// state canonical denies) refuse, because no backfill can make the log
|
||||
// un-claim a live row. This test stages exactly that incurable shape.
|
||||
const { brain } = await openBrain()
|
||||
await seedWrites(brain)
|
||||
const { kept } = await seedWrites(brain)
|
||||
await backfillBaseline(brain)
|
||||
await brain.flush()
|
||||
|
||||
// Age the brain: one canonical record the log never saw.
|
||||
const legacyId = '00000000-0000-4000-8000-00000000a6ed'
|
||||
// The log says `kept` is live; its canonical record vanishes behind the
|
||||
// write path's back (log-live-canonical-absent — the witness wins).
|
||||
const storage = internals(brain).storage
|
||||
await storage.saveNoun({
|
||||
id: legacyId,
|
||||
vector: new Array(384).fill(0.01),
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
})
|
||||
await storage.saveNounMetadata(legacyId, {
|
||||
noun: 'document',
|
||||
confidence: 0.5,
|
||||
createdAt: 1700000000000,
|
||||
updatedAt: 1700000000000,
|
||||
_rev: 1
|
||||
})
|
||||
await storage.writeNounRaw(kept, { metadata: null, vector: null })
|
||||
|
||||
let error: Error | null = null
|
||||
try {
|
||||
|
|
@ -248,9 +242,9 @@ describe('log authority — the switch, the oracle, the guarded flip', () => {
|
|||
} catch (err) {
|
||||
error = err as Error
|
||||
}
|
||||
expect(error, 'the flip rejects on a red oracle').not.toBeNull()
|
||||
expect(error!.message).toMatch(/oracle is RED/)
|
||||
expect(error!.message).toMatch(/baseline backfill/)
|
||||
expect(error, 'the flip rejects on a log-ahead divergence').not.toBeNull()
|
||||
expect(error!.message).toMatch(/witness denies/)
|
||||
expect(error!.message).toMatch(/log-live-canonical-absent/)
|
||||
|
||||
// Nothing changed: authority still tree, no artifact, deferred durability.
|
||||
expect(brain.logAuthority().authority).toBe('tree')
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@
|
|||
* @description Fact-log format v2 (record envelope + sector seals) pinned at
|
||||
* the byte level: every record type round-trips field-exact (bigint ints,
|
||||
* bin16 uuids, float-exact vectors), headers read v1 AND v2, unknown record
|
||||
* types/versions refuse loudly with the typed error, genesis width mismatches
|
||||
* types/versions refuse loudly with the typed error, the reserved crypto
|
||||
* envelope (cipherFlag/keyId — plaintext-only this release) refuses anything
|
||||
* nonzero/non-nil with the same typed error, genesis width mismatches
|
||||
* refuse naming both widths, sealed groups align to the sector size with
|
||||
* invisible pads, vector refs are writer-enforced single-hop, and torn tails
|
||||
* truncate to the intact prefix at EVERY byte offset. This module is the
|
||||
|
|
@ -11,7 +13,7 @@
|
|||
* vectors here are frozen; a change that breaks them is a format change.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { encode } from '@msgpack/msgpack'
|
||||
import { encode, decode } from '@msgpack/msgpack'
|
||||
import {
|
||||
encodeFactV2,
|
||||
decodeFact,
|
||||
|
|
@ -20,10 +22,13 @@ import {
|
|||
parseSegmentHeader,
|
||||
sealGroup,
|
||||
framePayload,
|
||||
encodePadFrame,
|
||||
minPadFrameBytes,
|
||||
UnknownLogRecordError,
|
||||
GenesisWidthMismatchError,
|
||||
LOG_RECORD_TYPES,
|
||||
LOG_RECORD_VERSION,
|
||||
LOG_RECORD_CIPHER_PLAINTEXT,
|
||||
FACT_LOG_FORMAT_V1,
|
||||
FACT_LOG_FORMAT_V2,
|
||||
SEGMENT_HEADER_BYTES,
|
||||
|
|
@ -254,7 +259,7 @@ describe('fact-log format v2 — golden byte vectors (frozen contract)', () => {
|
|||
records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }]
|
||||
})
|
||||
expect(hex(frame)).toBe(
|
||||
'2b000000c19ad9ff95cf0000000000000003cf0000018bcfe5687b91930201' +
|
||||
'2d00000048e4d43695cf0000000000000003cf0000018bcfe5687b9195020100c0' +
|
||||
'c41000000000000040008000000000000042c0c0'
|
||||
)
|
||||
})
|
||||
|
|
@ -370,11 +375,51 @@ describe('fact-log format v2 — decoder law (typed refusals, never skip)', () =
|
|||
})
|
||||
|
||||
it('a fact mixing known and unknown records still refuses (no partial reads)', () => {
|
||||
const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, uuidBytes(UUID(1))]
|
||||
const known = [LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))]
|
||||
const payload = encode([1, 1, [known, [200, 1]], null, null])
|
||||
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
|
||||
})
|
||||
|
||||
it('a nonzero cipherFlag refuses with the typed error — encrypted records need a newer reader', () => {
|
||||
const payload = encode(
|
||||
[1, 1, [[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 1, null, uuidBytes(UUID(1))]], null, null]
|
||||
)
|
||||
try {
|
||||
decodeFact(payload, 2)
|
||||
expect.unreachable('decode must throw')
|
||||
} catch (error) {
|
||||
const typed = error as UnknownLogRecordError
|
||||
expect(typed).toBeInstanceOf(UnknownLogRecordError)
|
||||
expect(typed.recordType).toBe(LOG_RECORD_TYPES.NOUN_TOMBSTONE)
|
||||
expect(typed.recordVersion).toBe(1)
|
||||
expect(typed.message).toMatch(/cipherFlag 1/)
|
||||
expect(typed.message).toMatch(/encrypted records need a newer reader/)
|
||||
}
|
||||
})
|
||||
|
||||
it('a non-nil keyId refuses the same way, even with cipherFlag 0', () => {
|
||||
const payload = encode(
|
||||
[
|
||||
1,
|
||||
1,
|
||||
[[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, uuidBytes(UUID(9)), uuidBytes(UUID(1))]],
|
||||
null,
|
||||
null
|
||||
]
|
||||
)
|
||||
expect(() => decodeFact(payload, 2)).toThrow(UnknownLogRecordError)
|
||||
expect(() => decodeFact(payload, 2)).toThrow(/encrypted records need a newer reader/)
|
||||
})
|
||||
|
||||
it('the encoder always writes the plaintext envelope: cipherFlag 0, keyId nil', () => {
|
||||
const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })))
|
||||
const raw = decode(payload) as unknown[]
|
||||
const record = (raw[2] as unknown[][])[0]
|
||||
expect(record[2]).toBe(LOG_RECORD_CIPHER_PLAINTEXT)
|
||||
expect(record[3]).toBeNull()
|
||||
expect(LOG_RECORD_CIPHER_PLAINTEXT).toBe(0)
|
||||
})
|
||||
|
||||
it('an unknown segment format version has no decode path', () => {
|
||||
const payload = framePayload(encodeFactV2(factOf(1, { type: 'noun.tombstone', id: UUID(1) })))
|
||||
expect(() => decodeFact(payload, 3)).toThrow(/reads 1 and 2/)
|
||||
|
|
@ -424,8 +469,8 @@ describe('fact-log format v2 — log.genesis width law', () => {
|
|||
1,
|
||||
1,
|
||||
[
|
||||
[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, uuidBytes(UUID(1))],
|
||||
[LOG_RECORD_TYPES.LOG_GENESIS, 1, 64, uuidBytes(UUID(9)), 1]
|
||||
[LOG_RECORD_TYPES.NOUN_TOMBSTONE, 1, 0, null, uuidBytes(UUID(1))],
|
||||
[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 64, uuidBytes(UUID(9)), 1]
|
||||
],
|
||||
null,
|
||||
null
|
||||
|
|
@ -434,7 +479,9 @@ describe('fact-log format v2 — log.genesis width law', () => {
|
|||
})
|
||||
|
||||
it('an invalid genesis width on the wire is malformed, not a mismatch', () => {
|
||||
const crafted = encode([1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 48, uuidBytes(UUID(9)), 1]], null, null])
|
||||
const crafted = encode(
|
||||
[1, 1, [[LOG_RECORD_TYPES.LOG_GENESIS, 1, 0, null, 48, uuidBytes(UUID(9)), 1]], null, null]
|
||||
)
|
||||
expect(() => decodeFact(crafted, 2)).toThrow(/32 or 64/)
|
||||
})
|
||||
})
|
||||
|
|
@ -504,7 +551,7 @@ describe('fact-log format v2 — vector legs (single-hop law)', () => {
|
|||
})
|
||||
expect(() => encodeFactV2(bad)).toThrow(/INLINE/)
|
||||
const craftedRef = encode(
|
||||
[1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, uuidBytes(UUID(7)), ['ref', 5]]], null, null]
|
||||
[1, 1, [[LOG_RECORD_TYPES.EMBED_LANDED, 1, 0, null, uuidBytes(UUID(7)), ['ref', 5]]], null, null]
|
||||
)
|
||||
expect(() => decodeFact(craftedRef, 2)).toThrow(/INLINE/)
|
||||
})
|
||||
|
|
@ -571,16 +618,30 @@ describe('fact-log format v2 — sector seals', () => {
|
|||
timestamp: 1_700_000_000_123,
|
||||
records: [{ type: 'noun.tombstone', id: '00000000-0000-4000-8000-000000000042' }]
|
||||
})
|
||||
const sealed = sealGroup([tomb], 64) // 51 bytes → gap 13 → overshoot → 77-byte pad
|
||||
const sealed = sealGroup([tomb], 64) // 53 bytes → gap 11 → overshoot → 75-byte pad
|
||||
expect(sealed.length).toBe(128)
|
||||
expect(hex(sealed.subarray(tomb.length))).toBe(
|
||||
// frame prefix + [0, 0, [[0, 1, bin8(42 zero bytes)]], nil, nil]
|
||||
'450000009463044d95cf0000000000000000cf000000000000000091930001c42a' +
|
||||
'0'.repeat(84) +
|
||||
// frame prefix + [0, 0, [[0, 1, bin8(40 zero bytes)]], nil, nil]
|
||||
'4300000088b4c8fa95cf0000000000000000cf000000000000000091930001c428' +
|
||||
'0'.repeat(80) +
|
||||
'c0c0'
|
||||
)
|
||||
})
|
||||
|
||||
it('encodePadFrame builds exact-size pads for streaming writers; refuses sub-minimum sizes', () => {
|
||||
// Pads are envelope-exempt (skipped wholesale), so the smallest pad frame
|
||||
// is byte-stable across the crypto-envelope change.
|
||||
expect(minPadFrameBytes()).toBe(33)
|
||||
for (const size of [minPadFrameBytes(), 64, 4096]) {
|
||||
const pad = encodePadFrame(size)
|
||||
expect(pad.length).toBe(size)
|
||||
const { facts: decoded, validBytes } = decodeGroupV2(pad)
|
||||
expect(decoded).toEqual([]) // invisible to readers
|
||||
expect(validBytes).toBe(size)
|
||||
}
|
||||
expect(() => encodePadFrame(minPadFrameBytes() - 1)).toThrow(/at least/)
|
||||
})
|
||||
|
||||
it('sealGroup refuses garbage: empty groups, malformed frames, bad seal sizes', () => {
|
||||
expect(() => sealGroup([], 4096)).toThrow(/at least one frame/)
|
||||
expect(() => sealGroup([new Uint8Array([1, 2, 3])], 4096)).toThrow(/not a well-formed frame/)
|
||||
|
|
@ -620,10 +681,12 @@ describe('fact-log format v2 — torn-tail discipline', () => {
|
|||
describe('fact-log format v2 — writer refusals (loud, never silent)', () => {
|
||||
const tombstone = (g: number): CommitFactV2 => factOf(g, { type: 'noun.tombstone', id: UUID(g) })
|
||||
|
||||
it('refuses empty records, generation 0, and a second batch.meta', () => {
|
||||
expect(() => encodeFactV2({ generation: 1, timestamp: 1, records: [] })).toThrow(
|
||||
/at least one record/
|
||||
)
|
||||
it('accepts empty records (an all-deduped batch is a real generation); refuses generation 0 and a second batch.meta', () => {
|
||||
// Contract change with the live cutover: v1 always encoded op-less
|
||||
// commits (a batch whose relates dedupe away still mints a generation);
|
||||
// v2 must not fork commit semantics — empty records round-trip.
|
||||
const empty = decodeFact(framePayload(encodeFactV2({ generation: 1, timestamp: 1, records: [] })), 2)
|
||||
expect(empty.records).toEqual([])
|
||||
expect(() => encodeFactV2({ ...tombstone(1), generation: 0 })).toThrow(/positive integer/)
|
||||
expect(() =>
|
||||
encodeFactV2({
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue