feat(8.0): generational MVCC storage + Datomic-style Db API (now/transact/asOf/with/persist)
One mechanism replaces the COW and versioning subsystems: immutable
generation-stamped records behind a Datomic-style database value.
Record layer (src/db/generationStore.ts):
- Monotonic generation counter in _system/generation.json; bumped once per
transact() commit and once per single-operation write (storage hook), so
brain.generation() is always a meaningful watermark.
- Commit protocol: stage before-images + tx.json delta -> fsync -> execute
batch via TransactionManager -> atomic tmp+rename of _system/manifest.json
(the rename IS the commit point) -> append _system/tx-log.jsonl.
- Crash recovery on open rolls uncommitted generations back byte-identically
and forces an index rebuild; refcounted pins gate compactHistory(), which
records a horizon (asOf below it throws GenerationCompactedError).
Db API:
- Db: get/find/search/related pinned at a generation, with() speculative
overlays, since() diffs, persist() hard-link-farm snapshots, timestamp,
generation, release() + FinalizationRegistry backstop.
- Brainy: now() O(1) pin, transact(ops, {meta, ifAtGeneration}) atomic batch
(GenerationConflictError CAS), asOf(generation|Date|path), restore(path,
{confirm}), compactHistory(), generation(), static open() + load().
- get()/metadata find()/related() are fully correct at any reachable pinned
generation; index-accelerated queries at historical generations throw
NotYetSupportedAtHistoricalGenerationError - never silently-wrong results.
- VersionedIndexProvider (generation/isGenerationVisible/pin/release) in
plugin.ts: feature-detected, balanced pin/release in lockstep with Db
lifecycle, post-commit applier + replay-gap model documented.
Storage primitives (BaseStorage + filesystem/memory adapters): raw-object
read/write/list/remove, fsync barrier, noun/verb raw before-image capture,
tx-log append, snapshotToDirectory (hard-link farm; byte-copy fallback and
append-in-place exceptions), restoreFromDirectory + derived-state reload.
Proof suite (tests/integration/db-mvcc.test.ts + tests/unit/db/): isolation
across 200 mutations, batch atomicity under injected execution failure,
ifAtGeneration CAS, snapshot immunity to source mutation, compaction safety
under pins, with() overlay isolation, generation monotonicity across
reopen, crash consistency through the real recovery path, and versioned
provider pin/release balance. Design record in
docs/ADR-001-generational-mvcc.md.
This commit is contained in:
parent
49e49483d1
commit
431cd64406
16 changed files with 6244 additions and 5 deletions
|
|
@ -3,6 +3,9 @@
|
|||
* In-memory storage adapter for environments where persistent storage is not available or needed
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs'
|
||||
import * as nodePath from 'node:path'
|
||||
import * as zlib from 'node:zlib'
|
||||
import {
|
||||
GraphVerb,
|
||||
HNSWNoun,
|
||||
|
|
@ -35,6 +38,11 @@ export class MemoryStorage extends BaseStorage {
|
|||
// storage has no local file, so getBinaryBlobPath() returns null.
|
||||
private blobStore: Map<string, Buffer> = new Map()
|
||||
|
||||
// Transaction log (8.0 MVCC) — the in-memory equivalent of the filesystem
|
||||
// adapter's `_system/tx-log.jsonl`. One JSON document per committed
|
||||
// transact(); serialized to a real JSONL file by snapshotToDirectory().
|
||||
private txLogLines: string[] = []
|
||||
|
||||
// Backward compatibility aliases
|
||||
private get metadata(): Map<string, any> {
|
||||
return this.objectStore
|
||||
|
|
@ -189,6 +197,132 @@ export class MemoryStorage extends BaseStorage {
|
|||
return null
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Generational record layer (8.0 MVCC) — tx-log + snapshot primitives
|
||||
// ===========================================================================
|
||||
|
||||
/**
|
||||
* Append one line to the in-memory transaction log (mirror of the
|
||||
* filesystem adapter's `_system/tx-log.jsonl` append).
|
||||
*
|
||||
* @param line - One complete JSON document, without trailing newline.
|
||||
*/
|
||||
public async appendTxLogLine(line: string): Promise<void> {
|
||||
this.txLogLines.push(line)
|
||||
}
|
||||
|
||||
/**
|
||||
* Read all transaction-log lines, oldest first (a copy — callers cannot
|
||||
* mutate the log).
|
||||
*/
|
||||
public async readTxLogLines(): Promise<string[]> {
|
||||
return [...this.txLogLines]
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the entire in-memory store to a directory in the exact layout
|
||||
* the filesystem adapter uses (uncompressed JSON objects, `_blobs/*.bin`
|
||||
* payloads, `_system/tx-log.jsonl`). The resulting directory is a
|
||||
* self-contained store openable via `Brainy.load(path)` — persisting an
|
||||
* in-memory brain produces a real, durable snapshot.
|
||||
*
|
||||
* @param targetPath - Absolute directory for the snapshot. Created if
|
||||
* missing; must be empty or absent (refuses to overwrite).
|
||||
*/
|
||||
public async snapshotToDirectory(targetPath: string): Promise<void> {
|
||||
try {
|
||||
const existing = await fs.promises.readdir(targetPath)
|
||||
if (existing.length > 0) {
|
||||
throw new Error(
|
||||
`snapshotToDirectory: target ${targetPath} already exists and is not empty. ` +
|
||||
`Choose a fresh directory per snapshot.`
|
||||
)
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') throw error
|
||||
}
|
||||
await fs.promises.mkdir(targetPath, { recursive: true })
|
||||
|
||||
for (const [key, value] of this.objectStore.entries()) {
|
||||
const filePath = nodePath.join(targetPath, ...key.split('/'))
|
||||
await fs.promises.mkdir(nodePath.dirname(filePath), { recursive: true })
|
||||
await fs.promises.writeFile(filePath, JSON.stringify(value, null, 2), 'utf-8')
|
||||
}
|
||||
|
||||
for (const [key, data] of this.blobStore.entries()) {
|
||||
const blobPath = nodePath.join(targetPath, '_blobs', ...key.split('/')) + '.bin'
|
||||
await fs.promises.mkdir(nodePath.dirname(blobPath), { recursive: true })
|
||||
await fs.promises.writeFile(blobPath, data)
|
||||
}
|
||||
|
||||
if (this.txLogLines.length > 0) {
|
||||
const logPath = nodePath.join(targetPath, '_system', 'tx-log.jsonl')
|
||||
await fs.promises.mkdir(nodePath.dirname(logPath), { recursive: true })
|
||||
await fs.promises.writeFile(logPath, this.txLogLines.map((l) => `${l}\n`).join(''), 'utf-8')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the in-memory store's contents from a snapshot directory —
|
||||
* accepts snapshots produced by either adapter (handles the filesystem
|
||||
* adapter's gzip-compressed `.json.gz` objects transparently), then reloads
|
||||
* all derived state.
|
||||
*
|
||||
* @param sourcePath - Absolute path of a snapshot directory.
|
||||
*/
|
||||
public async restoreFromDirectory(sourcePath: string): Promise<void> {
|
||||
const sourceStat = await fs.promises.stat(sourcePath).catch(() => null)
|
||||
if (!sourceStat || !sourceStat.isDirectory()) {
|
||||
throw new Error(`restoreFromDirectory: ${sourcePath} is not a directory`)
|
||||
}
|
||||
|
||||
this.objectStore.clear()
|
||||
this.blobStore.clear()
|
||||
this.txLogLines = []
|
||||
this.statistics = null
|
||||
|
||||
await this.loadDirectoryIntoStore(sourcePath, '')
|
||||
await this.reloadDerivedState()
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively load a snapshot directory into the object/blob/tx-log stores.
|
||||
* Lock directories from filesystem-adapter snapshots are skipped (process-
|
||||
* local state, meaningless in memory).
|
||||
*/
|
||||
private async loadDirectoryIntoStore(dirAbs: string, relPrefix: string): Promise<void> {
|
||||
const entries = await fs.promises.readdir(dirAbs, { withFileTypes: true })
|
||||
|
||||
for (const entry of entries) {
|
||||
const rel = relPrefix ? `${relPrefix}/${entry.name}` : entry.name
|
||||
const abs = nodePath.join(dirAbs, entry.name)
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
if (relPrefix === '' && entry.name === 'locks') continue
|
||||
await this.loadDirectoryIntoStore(abs, rel)
|
||||
continue
|
||||
}
|
||||
if (!entry.isFile()) continue
|
||||
|
||||
if (rel === '_system/tx-log.jsonl') {
|
||||
const content = await fs.promises.readFile(abs, 'utf-8')
|
||||
this.txLogLines = content.split('\n').filter((l) => l.length > 0)
|
||||
} else if (relPrefix.startsWith('_blobs') && rel.endsWith('.bin')) {
|
||||
const key = rel.slice('_blobs/'.length, -'.bin'.length)
|
||||
this.blobStore.set(key, await fs.promises.readFile(abs))
|
||||
} else if (rel.endsWith('.json.gz') || rel.endsWith('.gz')) {
|
||||
const raw = await fs.promises.readFile(abs)
|
||||
const key = rel.slice(0, -'.gz'.length)
|
||||
this.objectStore.set(key, JSON.parse(zlib.gunzipSync(raw).toString('utf-8')))
|
||||
} else if (entry.name.includes('.tmp.')) {
|
||||
// In-flight atomic-write remnant from a crashed filesystem store — skip.
|
||||
} else {
|
||||
const content = await fs.promises.readFile(abs, 'utf-8')
|
||||
this.objectStore.set(rel, JSON.parse(content))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get multiple metadata objects in batches (CRITICAL: Prevents socket exhaustion)
|
||||
* Memory storage implementation is simple since all data is already in memory
|
||||
|
|
@ -217,6 +351,7 @@ export class MemoryStorage extends BaseStorage {
|
|||
public async clear(): Promise<void> {
|
||||
this.objectStore.clear()
|
||||
this.blobStore.clear()
|
||||
this.txLogLines = []
|
||||
this.statistics = null
|
||||
this.totalNounCount = 0
|
||||
this.totalVerbCount = 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue