feat: generation fact log — after-image commit records, dual-written at every commit point
Every committed generation now also appends a FACT — an after-image commit record (what each touched entity/relationship became, or a body-less tombstone for a removal) — to an append-only, crc32c-framed segment log under _generations/facts/. The before-image history and the canonical tree remain authoritative; the fact log gives consumers ONE sequential, self-verifying stream (index heals, incremental replays) in place of a per-entity directory walk. - Wire format: positional msgpack facts [generation, timestamp, ops, meta, blobHashes]; op = [kind u8, id bin16, record | nil tombstone]; 32-byte segment header (magic, formatVersion, firstGeneration, zeroed+verified reserved); length+crc32c frame per fact; zero-padded segment names so lexicographic order == generation order; JSON manifest with an atomic rename flip, manifest-first rotation. - Commit protocol: facts append+fsync BEFORE the commit point inside the existing durability window, so a crash can only leave the log AHEAD of committed truth — open() truncates back (torn tails detected by CRC). Absent generation = never committed; a scan can never see an uncommitted fact. transact() facts are durable-on-return; single-op facts ride the group-commit flush exactly like buffered history. A fact-append failure fails the write, loudly — a silent gap would be a lie a later replay discovers. - New public surface: brain.scanFacts() (sequential batches with heal telemetry: head/segments/approx up front, per-batch generation range + bytes + segment id, loud abort on gaps, summary cross-check) and brain.factSegmentPaths() (immutable sealed segments for zero-copy consumers; the mutable tail excluded). Exported types CommitFact, FactOp, FactScanBatch, FactScanHandle. - Storage: optional binary raw-byte primitives (appendRawBytes, readRawBytes, writeRawBytes, rawByteSize) on StorageAdapter — feature-detected; filesystem + memory adapters implement them; an adapter without them hosts no fact log. Fact segments are byte-copied (never hard-linked) into snapshots. The _generations/facts/ namespace is registered as a protected family (rebuildable: false): no sweeper or GC may delete under it. - New crc32c (Castagnoli) utility with RFC known-answer tests.
This commit is contained in:
parent
92299f27be
commit
38b0041464
13 changed files with 1493 additions and 4 deletions
|
|
@ -693,6 +693,17 @@ export class FileSystemStorage extends BaseStorage {
|
|||
*/
|
||||
private static readonly SNAPSHOT_BYTE_COPY_DIRS = new Set<string>(['_id_mapper'])
|
||||
|
||||
/**
|
||||
* Nested path PREFIXES whose files are byte-copied into snapshots, not
|
||||
* hard-linked — for append-in-place files below the top level. The
|
||||
* generation fact log's tail segment is appended in place between rotations;
|
||||
* a hard-linked tail would let post-snapshot appends reach through into the
|
||||
* snapshot. (Sealed segments are immutable and would be link-safe, but the
|
||||
* prefix rule keeps the discipline simple; segments are bounded by the
|
||||
* rotation threshold, so the copy cost is small.)
|
||||
*/
|
||||
private static readonly SNAPSHOT_BYTE_COPY_PREFIXES: string[] = ['_generations/facts/']
|
||||
|
||||
/**
|
||||
* Top-level directories excluded from snapshots: process-local lock state
|
||||
* (writer lock, flush-request RPC files) must never travel with the data, and
|
||||
|
|
@ -870,6 +881,75 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
// ==========================================================================
|
||||
// Binary raw-byte primitives — the substrate for append-only log-structured
|
||||
// files (the generation fact log's CRC-framed segments). Paths are used
|
||||
// VERBATIM (no .gz/.bin suffixing). Append durability rides syncRawObjects
|
||||
// at the commit barrier, like every other staged write.
|
||||
// ==========================================================================
|
||||
|
||||
/**
|
||||
* Append bytes to a raw binary file, creating it (and parent directories)
|
||||
* when absent. NOT fsync'd here — the caller batches durability via
|
||||
* `syncRawObjects` at its commit barrier.
|
||||
*/
|
||||
public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
const fullPath = path.join(this.rootDir, rawPath)
|
||||
await fs.promises.mkdir(path.dirname(fullPath), { recursive: true })
|
||||
await fs.promises.appendFile(fullPath, bytes)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a raw binary file whole. Absent → `null`; a real IO fault throws —
|
||||
* a present-but-unreadable log segment must never read as "no facts".
|
||||
*/
|
||||
public async readRawBytes(rawPath: string): Promise<Uint8Array | null> {
|
||||
await this.ensureInitialized()
|
||||
try {
|
||||
const buf: Buffer = await fs.promises.readFile(path.join(this.rootDir, rawPath))
|
||||
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
|
||||
} catch (error: any) {
|
||||
if (isAbsentError(error)) return null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace a raw binary file atomically: write-new → fsync → rename. The
|
||||
* reconcile primitive (e.g. truncating a fact-log tail back to committed
|
||||
* truth after a crash) — a crash mid-replace leaves either the old file or
|
||||
* the new one, never a mix.
|
||||
*/
|
||||
public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
const fullPath = path.join(this.rootDir, rawPath)
|
||||
await fs.promises.mkdir(path.dirname(fullPath), { recursive: true })
|
||||
const tmpPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).slice(2)}`
|
||||
const handle = await fs.promises.open(tmpPath, 'w')
|
||||
try {
|
||||
await handle.writeFile(bytes)
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
await fs.promises.rename(tmpPath, fullPath)
|
||||
}
|
||||
|
||||
/**
|
||||
* Byte size of a raw binary file, or `null` when absent.
|
||||
*/
|
||||
public async rawByteSize(rawPath: string): Promise<number | null> {
|
||||
await this.ensureInitialized()
|
||||
try {
|
||||
const stat = await fs.promises.stat(path.join(this.rootDir, rawPath))
|
||||
return stat.size
|
||||
} catch (error: any) {
|
||||
if (isAbsentError(error)) return null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Snapshot the entire store into `targetPath` as a hard-link farm
|
||||
* (Cassandra-style: instant, space-shared). Safe because every data file
|
||||
|
|
@ -919,7 +999,8 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// msync/truncate reach through into the snapshot.
|
||||
if (
|
||||
FileSystemStorage.SNAPSHOT_BYTE_COPY_PATHS.has(normalized) ||
|
||||
FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0])
|
||||
FileSystemStorage.SNAPSHOT_BYTE_COPY_DIRS.has(normalized.split('/')[0]) ||
|
||||
FileSystemStorage.SNAPSHOT_BYTE_COPY_PREFIXES.some((p) => normalized.startsWith(p))
|
||||
) {
|
||||
await fs.promises.copyFile(sourceFile, targetFile)
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -222,6 +222,45 @@ export class MemoryStorage extends BaseStorage {
|
|||
return [...this.txLogLines]
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Binary raw-byte primitives — in-memory mirror of the filesystem adapter's
|
||||
// append-only substrate (the generation fact log's segments), so memory
|
||||
// brains dual-write facts too and the compat suite runs on both adapters.
|
||||
// ===========================================================================
|
||||
|
||||
/** Raw binary files, keyed by verbatim path. */
|
||||
private rawBytesStore: Map<string, Uint8Array> = new Map()
|
||||
|
||||
/** Append bytes to a raw binary file, creating it when absent. */
|
||||
public async appendRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
|
||||
const existing = this.rawBytesStore.get(rawPath)
|
||||
if (!existing) {
|
||||
this.rawBytesStore.set(rawPath, bytes.slice())
|
||||
return
|
||||
}
|
||||
const merged = new Uint8Array(existing.length + bytes.length)
|
||||
merged.set(existing, 0)
|
||||
merged.set(bytes, existing.length)
|
||||
this.rawBytesStore.set(rawPath, merged)
|
||||
}
|
||||
|
||||
/** Read a raw binary file whole (a copy); absent → null. */
|
||||
public async readRawBytes(rawPath: string): Promise<Uint8Array | null> {
|
||||
const bytes = this.rawBytesStore.get(rawPath)
|
||||
return bytes ? bytes.slice() : null
|
||||
}
|
||||
|
||||
/** Replace a raw binary file (atomic by construction in memory). */
|
||||
public async writeRawBytes(rawPath: string, bytes: Uint8Array): Promise<void> {
|
||||
this.rawBytesStore.set(rawPath, bytes.slice())
|
||||
}
|
||||
|
||||
/** Byte size of a raw binary file, or null when absent. */
|
||||
public async rawByteSize(rawPath: string): Promise<number | null> {
|
||||
const bytes = this.rawBytesStore.get(rawPath)
|
||||
return bytes ? bytes.length : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the entire in-memory store to a directory in the exact layout
|
||||
* the filesystem adapter uses (uncompressed JSON objects, `_blobs/*.bin`
|
||||
|
|
@ -354,6 +393,7 @@ export class MemoryStorage extends BaseStorage {
|
|||
public async clear(): Promise<void> {
|
||||
this.objectStore.clear()
|
||||
this.blobStore.clear()
|
||||
this.rawBytesStore.clear()
|
||||
this.txLogLines = []
|
||||
this.statistics = null
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue