perf(8.0): bound per-id generation history chains (O(W+L) resident, was O(N))
The MVCC time-travel layer kept nounChains/verbChains as unbounded Map<id, number[]> — one resident chain per id ever touched across retained history (O(N) RAM, defeating billion-scale time travel). Replace with a hot-tail window + bounded cold LRU + a mutex-free bulk resolver, keeping resolveAt exactly correct. The most-recent W generations stay resident as full chains (the common recent-pin read is O(log), zero scan); deeper pins reconstruct one id's chain on demand into an L-bounded LRU. Reconstruction is LOCK-LIGHT — it holds no commit mutex, so a historical read never stalls writers; correctness holds because a live pin's answer is always > minPinnedGeneration, which compaction can never reclaim, and a concurrently-reclaimed gen below that is skipped. materializeAtGeneration routes through a new mutex-free resolveManyAt (one forward pass, O(R + |ids|)) — without it a deep-pin materialize both regresses to O(N*R) and deadlocks on snapshotWith's mutex. The cold cache is invalidated on re-touch to stay coherent. Resident RAM is O(W*d + L), independent of N. 11-case test (oracle-vs-bruteforce, held-Db across eviction + concurrent compaction, no-deadlock, write-path I/O-free); unit 1735 + integration 613 (db-mvcc/db-temporal/db-asof green).
This commit is contained in:
parent
8f4787bb10
commit
a859d6ecf8
4 changed files with 956 additions and 72 deletions
|
|
@ -7475,23 +7475,42 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const snapshotStorage = new MemoryStorage()
|
||||
await snapshotStorage.init()
|
||||
|
||||
/** Copy one id's at-`generation` state into the snapshot (or ensure absence). */
|
||||
const copyAt = async (kind: 'noun' | 'verb', id: string): Promise<void> => {
|
||||
const resolved = await this.generationStore.resolveAt(kind, id, generation)
|
||||
// Copy one id's at-`generation` state into the snapshot (or ensure absence)
|
||||
// as a PURE LOOKUP against a precomputed `firstAfter` map (id → the first
|
||||
// generation after the pin that touched it). An id absent from the map was
|
||||
// untouched since the pin → its live storage state IS its at-`generation`
|
||||
// state; an id present resolves from that generation's immutable
|
||||
// before-image. Both `resolveManyAt` (builds the map) and
|
||||
// `readGenerationRecord` (reads the before-image) are MUTEX-FREE, so this is
|
||||
// safe to call inside the reconciliation pass below (which holds the commit
|
||||
// mutex via `snapshotWith`) — a per-id `resolveAt` there would re-enter the
|
||||
// mutex and DEADLOCK, and would regress the O(R) bulk copy to O(N·R).
|
||||
const copyAt = async (
|
||||
kind: 'noun' | 'verb',
|
||||
id: string,
|
||||
firstAfter: Map<string, number>
|
||||
): Promise<void> => {
|
||||
let record: { metadata: any; vector: any | null } | null = null
|
||||
if (resolved.source === 'record') {
|
||||
if (resolved.metadata !== null) {
|
||||
record = { metadata: resolved.metadata, vector: resolved.vector }
|
||||
}
|
||||
} else if (resolved.source === 'current') {
|
||||
const candidate = firstAfter.get(id)
|
||||
if (candidate === undefined) {
|
||||
// Untouched since the pin → live storage IS the at-`generation` state.
|
||||
const raw =
|
||||
kind === 'noun' ? await this.storage.readNounRaw(id) : await this.storage.readVerbRaw(id)
|
||||
if (raw.metadata !== null) {
|
||||
record = raw
|
||||
if (raw.metadata !== null) record = raw
|
||||
} else {
|
||||
const before = await this.generationStore.readGenerationRecord(kind, candidate, id)
|
||||
if (before === null) {
|
||||
throw new Error(
|
||||
`Generation record missing: _generations/${candidate}/prev/${id}.json ` +
|
||||
`(store corrupted or records removed outside compactHistory())`
|
||||
)
|
||||
}
|
||||
// A non-null metadata before-image is the at-`generation` live state; a
|
||||
// null-metadata before-image (the create sentinel, or a metadata-less
|
||||
// stored state) is absent at this generation.
|
||||
if (before.metadata !== null) record = { metadata: before.metadata, vector: before.vector }
|
||||
}
|
||||
// `record === null` — absent at this generation (or a metadata-less
|
||||
// stored state, which is not a live entity): writing nulls ensures the
|
||||
// `record === null` — absent at this generation: writing nulls ensures the
|
||||
// id is absent in the snapshot, which also un-copies ids created by
|
||||
// transactions that commit mid-build.
|
||||
if (kind === 'noun') {
|
||||
|
|
@ -7516,25 +7535,42 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const id = entityIdFromCanonicalPath(path)
|
||||
if (id) verbIds.add(id)
|
||||
}
|
||||
for (const id of nounIds) await copyAt('noun', id)
|
||||
for (const id of verbIds) await copyAt('verb', id)
|
||||
// ONE ascending pass per kind resolves the whole universe's first-after
|
||||
// generations (O(R) getDelta reads, NOT O(N·R)), then each copy is a lookup.
|
||||
const nounFirstAfter = await this.generationStore.resolveManyAt('noun', nounIds, generation)
|
||||
const verbFirstAfter = await this.generationStore.resolveManyAt('verb', verbIds, generation)
|
||||
for (const id of nounIds) await copyAt('noun', id, nounFirstAfter)
|
||||
for (const id of verbIds) await copyAt('verb', id, verbFirstAfter)
|
||||
|
||||
// Reconciliation pass under the commit mutex: nothing can commit during
|
||||
// this section, so afterwards the snapshot is exactly the at-`generation`
|
||||
// record set even when transactions committed while the bulk copy ran.
|
||||
// Reconciled ids join the enumeration sets so the vector-index build
|
||||
// below sees them too.
|
||||
// below sees them too. `resolveManyAt`/`readGenerationRecord` are mutex-free,
|
||||
// so this section never re-enters the commit mutex (no deadlock).
|
||||
await this.generationStore.snapshotWith(async () => {
|
||||
const committedNow = this.generationStore.committedGeneration()
|
||||
if (committedNow === watermark) return
|
||||
const delta = await this.generationStore.changedBetween(watermark, committedNow)
|
||||
for (const id of delta.nouns) {
|
||||
const reconNouns = new Set<string>(delta.nouns)
|
||||
const reconVerbs = new Set<string>(delta.verbs)
|
||||
const reconNounFirstAfter = await this.generationStore.resolveManyAt(
|
||||
'noun',
|
||||
reconNouns,
|
||||
generation
|
||||
)
|
||||
const reconVerbFirstAfter = await this.generationStore.resolveManyAt(
|
||||
'verb',
|
||||
reconVerbs,
|
||||
generation
|
||||
)
|
||||
for (const id of reconNouns) {
|
||||
nounIds.add(id)
|
||||
await copyAt('noun', id)
|
||||
await copyAt('noun', id, reconNounFirstAfter)
|
||||
}
|
||||
for (const id of delta.verbs) {
|
||||
for (const id of reconVerbs) {
|
||||
verbIds.add(id)
|
||||
await copyAt('verb', id)
|
||||
await copyAt('verb', id, reconVerbFirstAfter)
|
||||
}
|
||||
watermark = committedNow
|
||||
})
|
||||
|
|
|
|||
|
|
@ -147,19 +147,80 @@ export class GenerationStore {
|
|||
* ({@link committedRanges})
|
||||
* (which is O(database-age) — confirmed by the scalability spike: a read of an
|
||||
* unchanged entity at an old pin scaled 11.9x for 10x history depth). This
|
||||
* mirrors cor's `delta_history` chain. Built lazily once (under the mutex),
|
||||
* maintained on commit, invalidated on compaction.
|
||||
* mirrors cor's `delta_history` chain.
|
||||
*
|
||||
* BOUNDED-RAM design (Approach C — hot-tail window + cold LRU). The old
|
||||
* unbounded `Map<id, number[]>` held ONE resident chain per id ever touched
|
||||
* across retained history → O(distinct-ids-ever-touched) RAM, which defeats
|
||||
* billion-scale time travel (the chains, not the data, become the heap floor).
|
||||
* These three structures replace it with RAM bounded by the knobs W/L,
|
||||
* INDEPENDENT of the corpus size:
|
||||
*
|
||||
* 1. {@link recentNounChains}/{@link recentVerbChains} — FULL per-id chains,
|
||||
* but only over the HOT TAIL window `[windowLo, counter]` (the last
|
||||
* {@link recentWindowGenerations} generations). Pins at-or-above
|
||||
* `windowLo` (the overwhelming common case — recent history) resolve here
|
||||
* with the same O(log chain) binary search as before.
|
||||
* 2. {@link coldNounChains}/{@link coldVerbChains} — a bounded LRU of FULL
|
||||
* per-id chains for DEEP pins (`gen < windowLo`). Reconstructed on demand
|
||||
* from the persisted deltas ({@link reconstructColdChain}), cached so a
|
||||
* re-read is O(log chain), evicted oldest-first past
|
||||
* {@link coldChainLruMax}. Empty chains are cached too (bounded negative
|
||||
* cache for untouched-in-cold-region ids).
|
||||
* 3. {@link windowNounDeltas}/{@link windowVerbDeltas} — the window's INVERSE
|
||||
* index (`gen → ids touched at that gen`), fed from the touched sets
|
||||
* already handed to {@link extendChains}. It makes sliding `windowLo`
|
||||
* forward an O(d̄) front-trim of exactly the ids leaving the window, with
|
||||
* ZERO disk I/O — the write path never reads `tx.json` on a slide.
|
||||
*
|
||||
* INVARIANT — purely derived, NEVER persisted. All three structures (and the
|
||||
* cold LRU) are reconstructable from the on-disk deltas at any time; eviction
|
||||
* (cold LRU) and window slide-out drop entries that are then rebuilt on the
|
||||
* next read that needs them. Correctness across eviction/reconstruction is
|
||||
* guaranteed by pin-gating: a live historical `Db` pins at `g_old`, and the
|
||||
* before-image a read needs lives at the FIRST generation after `g_old`
|
||||
* (`> g_old ≥ minPinned`), which compaction can never reclaim while the pin is
|
||||
* held — so a reconstructed chain yields the identical answer as the original.
|
||||
*/
|
||||
private readonly nounChains = new Map<string, number[]>()
|
||||
private readonly verbChains = new Map<string, number[]>()
|
||||
/** Whether {@link nounChains}/{@link verbChains} reflect every committed generation. */
|
||||
private chainsReady = false
|
||||
/** De-dupes concurrent first-callers of {@link ensureChains}. */
|
||||
private chainsBuilding: Promise<void> | null = null
|
||||
private readonly recentNounChains = new Map<string, number[]>()
|
||||
private readonly recentVerbChains = new Map<string, number[]>()
|
||||
/** Window inverse index (`gen → ids touched`), one map per kind — the O(d̄),
|
||||
* I/O-free slide-trim driver. Holds exactly the gens in `[windowLo, counter]`. */
|
||||
private readonly windowNounDeltas = new Map<number, string[]>()
|
||||
private readonly windowVerbDeltas = new Map<number, string[]>()
|
||||
/** Bounded LRU of reconstructed deep-pin chains (`gen < windowLo`). JS Map
|
||||
* insertion order is the LRU order: a hit re-inserts (delete+set) to mark
|
||||
* MRU; eviction removes `keys().next().value` (the oldest) past the cap. */
|
||||
private readonly coldNounChains = new Map<string, number[]>()
|
||||
private readonly coldVerbChains = new Map<string, number[]>()
|
||||
/** Inclusive lower bound of the resident hot-tail window (`0` until built). */
|
||||
private windowLo = 0
|
||||
/** Whether the hot-tail window + its inverse index are built and current. */
|
||||
private windowReady = false
|
||||
/** De-dupes concurrent first-callers of {@link ensureWindow}. */
|
||||
private windowBuilding: Promise<void> | null = null
|
||||
|
||||
/**
|
||||
* Hot-tail window width W — how many of the newest generations keep a resident
|
||||
* per-id chain ({@link recentNounChains}). Bounds recent-chain + window-delta
|
||||
* RAM to O(W·d̄), independent of corpus size. Deep pins below the window fall to
|
||||
* the cold LRU. Not `readonly` so tests can shrink it to exercise the
|
||||
* cold/slide paths without building thousands of generations.
|
||||
*/
|
||||
private recentWindowGenerations = 4096
|
||||
|
||||
/**
|
||||
* Cold-chain LRU capacity L — how many reconstructed deep-pin chains stay
|
||||
* resident. Bounds cold-chain RAM to O(L·avg-chain-length), independent of how
|
||||
* many distinct ids are deep-read. Not `readonly` so tests can shrink it to
|
||||
* exercise eviction + reconstruction.
|
||||
*/
|
||||
private coldChainLruMax = 4096
|
||||
|
||||
/**
|
||||
* Cap on resident {@link deltaCache} entries (Model-B RAM bound). The per-id
|
||||
* {@link nounChains}/{@link verbChains} are the hot read structure; the raw
|
||||
* {@link recentNounChains}/{@link recentVerbChains} window is the hot read
|
||||
* structure; the raw
|
||||
* deltas are only needed for range queries ({@link changedBetween}, `diff`,
|
||||
* `since`) and chain (re)builds, and {@link getDelta} transparently re-reads an
|
||||
* evicted delta from storage. Bounding this keeps a long-lived high-write
|
||||
|
|
@ -290,8 +351,8 @@ export class GenerationStore {
|
|||
// committedRanges only includes generations ≤ the manifest watermark.
|
||||
this.counter = Math.max(this.counter, gen)
|
||||
}
|
||||
// Chains are rebuilt lazily from the (possibly changed) generation set on
|
||||
// the next historical read — covers reopen and reopen-after-restore.
|
||||
// The history window is rebuilt lazily from the (possibly changed) generation
|
||||
// set on the next historical read — covers reopen and reopen-after-restore.
|
||||
this.invalidateChains()
|
||||
|
||||
if (rolledBack > 0) {
|
||||
|
|
@ -1016,16 +1077,39 @@ export class GenerationStore {
|
|||
| { source: 'absent' }
|
||||
| { source: 'record'; metadata: any; vector: any | null }
|
||||
> {
|
||||
await this.ensureChains()
|
||||
const chain = (kind === 'noun' ? this.nounChains : this.verbChains).get(id)
|
||||
if (chain === undefined) {
|
||||
// The id was never touched by any committed generation → the live
|
||||
// storage state IS its state at `gen`.
|
||||
return { source: 'current' }
|
||||
await this.ensureWindow()
|
||||
// Snapshot windowLo, then read the chain synchronously (no await between):
|
||||
// a concurrent commit can only slide the window at the `await` above, so the
|
||||
// routing decision and the chain it reads are a consistent pair.
|
||||
const windowLo = this.windowLo
|
||||
let candidate: number | undefined
|
||||
if (gen >= windowLo) {
|
||||
// HOT path — every generation that could hold the before-image
|
||||
// (`> gen ≥ windowLo`) is inside the resident window, so the recent chain
|
||||
// is complete for this pin. O(log chain).
|
||||
const chain = (kind === 'noun' ? this.recentNounChains : this.recentVerbChains).get(id)
|
||||
if (chain === undefined) return { source: 'current' }
|
||||
candidate = firstGenerationAfter(chain, gen)
|
||||
} else {
|
||||
// COLD path — a deep pin below the window. Serve from the bounded LRU,
|
||||
// reconstructing (lock-light) on a miss. Caching empty chains too keeps
|
||||
// repeated deep reads of untouched-in-cold-region ids O(1).
|
||||
const cold = kind === 'noun' ? this.coldNounChains : this.coldVerbChains
|
||||
let chain = cold.get(id)
|
||||
if (chain !== undefined) {
|
||||
cold.delete(id) // re-insert to mark most-recently-used
|
||||
cold.set(id, chain)
|
||||
} else {
|
||||
chain = await this.reconstructColdChain(kind, id)
|
||||
cold.set(id, chain)
|
||||
while (cold.size > this.coldChainLruMax) {
|
||||
const oldest = cold.keys().next().value
|
||||
if (oldest === undefined) break
|
||||
cold.delete(oldest)
|
||||
}
|
||||
}
|
||||
candidate = firstGenerationAfter(chain, gen)
|
||||
}
|
||||
// The first committed generation strictly greater than `gen` that touched
|
||||
// `id` holds the before-image of its state AS OF `gen` — O(log chain).
|
||||
const candidate = firstGenerationAfter(chain, gen)
|
||||
if (candidate === undefined) {
|
||||
// Nothing after `gen` touched `id`; the live state is its state at `gen`.
|
||||
return { source: 'current' }
|
||||
|
|
@ -1045,6 +1129,87 @@ export class GenerationStore {
|
|||
return { source: 'record', metadata: record.metadata, vector: record.vector }
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Resolve the FIRST reserved generation after `gen` that touched
|
||||
* each of `ids`, by kind — the bulk, MUTEX-FREE counterpart of
|
||||
* {@link resolveAt}'s chain lookup. A SINGLE ascending pass over the reserved
|
||||
* (committed ∪ pending) generations records `firstAfter[id] = g` for the first
|
||||
* `g > gen` whose delta touched `id`, stopping early once every id is resolved.
|
||||
*
|
||||
* Ids absent from the returned map were never touched after `gen` → their live
|
||||
* storage state IS their state at `gen` (the {@link resolveAt} `'current'`
|
||||
* answer). Ids present map to the generation whose before-image
|
||||
* ({@link readGenerationRecord}) holds their state as of `gen`.
|
||||
*
|
||||
* WHY MUTEX-FREE (load-bearing): {@link Brainy.materializeAtGeneration} runs its
|
||||
* final reconciliation inside {@link snapshotWith} (which HOLDS the commit
|
||||
* mutex). Resolving ids one-by-one through {@link resolveAt} there would (a)
|
||||
* re-enter the mutex via {@link ensureWindow} → DEADLOCK, and (b) regress an
|
||||
* O(R) bulk copy to O(N·R). This method takes no lock and makes one pass, so a
|
||||
* deep-pin materialize over N ids stays O(R·d̄ + |ids|) with O(R) `getDelta`
|
||||
* calls. It iterates SNAPSHOTS of the small committed-range list (O(gaps)) and
|
||||
* the pending list (O(pending)) — NOT a materialized O(R) gen array — so it is
|
||||
* safe against concurrent commits and bounded in memory.
|
||||
*
|
||||
* @param kind - `'noun'` for entities, `'verb'` for relationships.
|
||||
* @param ids - The ids to resolve.
|
||||
* @param gen - The pinned generation.
|
||||
* @returns `id → first reserved generation after `gen` that touched it` for the
|
||||
* subset of `ids` touched after `gen`.
|
||||
*/
|
||||
async resolveManyAt(
|
||||
kind: 'noun' | 'verb',
|
||||
ids: Set<string>,
|
||||
gen: number
|
||||
): Promise<Map<string, number>> {
|
||||
const firstAfter = new Map<string, number>()
|
||||
if (ids.size === 0) return firstAfter
|
||||
// Snapshot the small range list + pending list (NOT the expanded gens) so the
|
||||
// pass is immune to a concurrent compaction splice / commit append and stays
|
||||
// O(gaps + pending) in memory.
|
||||
const ranges = this.committedRanges.map(([s, e]) => [s, e] as [number, number])
|
||||
const pending = [...this.pendingGens]
|
||||
const record = async (g: number): Promise<boolean> => {
|
||||
const delta = await this.getDelta(g)
|
||||
const touched = kind === 'noun' ? delta.nouns : delta.verbs
|
||||
for (const id of touched) {
|
||||
if (ids.has(id) && !firstAfter.has(id)) firstAfter.set(id, g)
|
||||
}
|
||||
return firstAfter.size === ids.size
|
||||
}
|
||||
for (const [s, e] of ranges) {
|
||||
if (e <= gen) continue
|
||||
for (let g = Math.max(s, gen + 1); g <= e; g++) {
|
||||
if (await record(g)) return firstAfter
|
||||
}
|
||||
}
|
||||
for (const g of pending) {
|
||||
if (g <= gen) continue
|
||||
if (await record(g)) return firstAfter
|
||||
}
|
||||
return firstAfter
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the before-image of `id` recorded by generation `gen` — the
|
||||
* public, MUTEX-FREE accessor {@link Brainy.materializeAtGeneration}'s `copyAt`
|
||||
* uses after {@link resolveManyAt} hands it the generation. Pairs with
|
||||
* `resolveManyAt` so the bulk historical copy never touches the commit mutex
|
||||
* (no deadlock inside {@link snapshotWith}, no per-id chain rebuild).
|
||||
* @param kind - `'noun'` for entities, `'verb'` for relationships.
|
||||
* @param gen - The generation whose before-image to read.
|
||||
* @param id - The id to read.
|
||||
* @returns The stored {@link GenerationRecord}, or `null` when the record-set
|
||||
* has no before-image for `id`.
|
||||
*/
|
||||
async readGenerationRecord(
|
||||
kind: 'noun' | 'verb',
|
||||
gen: number,
|
||||
id: string
|
||||
): Promise<GenerationRecord | null> {
|
||||
return this.readBeforeImage(kind, gen, id)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the before-image of `id` stored by generation `gen` from
|
||||
* the right tier: the in-memory pending buffer while the generation is
|
||||
|
|
@ -1066,50 +1231,176 @@ export class GenerationStore {
|
|||
}
|
||||
|
||||
/**
|
||||
* @description Ensure the per-id history chains ({@link nounChains} /
|
||||
* {@link verbChains}) reflect every committed generation. Built once, lazily,
|
||||
* under the commit mutex (so no concurrent commit mutates {@link committedRanges}
|
||||
* mid-build); thereafter maintained incrementally by {@link commit} and
|
||||
* invalidated by {@link compact}. Concurrent first-callers share one build.
|
||||
* O(committed generations) the first time; O(1) afterwards.
|
||||
* @description Build the resident hot-tail window — the per-id chains
|
||||
* ({@link recentNounChains}/{@link recentVerbChains}) and the inverse index
|
||||
* ({@link windowNounDeltas}/{@link windowVerbDeltas}) over the last
|
||||
* {@link recentWindowGenerations} reserved generations, `[windowLo, counter]`.
|
||||
* Built once, lazily, under the commit mutex (so no concurrent commit mutates
|
||||
* {@link committedRanges}/pending mid-build); thereafter maintained
|
||||
* incrementally by {@link extendChains} and invalidated by {@link compact} /
|
||||
* reopen. Concurrent first-callers share one build. O(W) `getDelta` reads the
|
||||
* first time (only the window, NOT all of history); O(1) afterwards. Deep pins
|
||||
* below `windowLo` are served by {@link reconstructColdChain} on demand.
|
||||
*/
|
||||
private async ensureChains(): Promise<void> {
|
||||
if (this.chainsReady) return
|
||||
if (!this.chainsBuilding) {
|
||||
this.chainsBuilding = this.withMutex(async () => {
|
||||
if (this.chainsReady) return
|
||||
this.nounChains.clear()
|
||||
this.verbChains.clear()
|
||||
// reservedGensAsc (committed ∪ pending) is sorted ascending, so chains
|
||||
// accrue in ascending order and un-flushed single-ops are indexed too.
|
||||
private async ensureWindow(): Promise<void> {
|
||||
if (this.windowReady) return
|
||||
if (!this.windowBuilding) {
|
||||
this.windowBuilding = this.withMutex(async () => {
|
||||
if (this.windowReady) return
|
||||
this.recentNounChains.clear()
|
||||
this.recentVerbChains.clear()
|
||||
this.windowNounDeltas.clear()
|
||||
this.windowVerbDeltas.clear()
|
||||
// The window is the newest W reserved generations, clamped above the
|
||||
// compaction horizon (gens ≤ horizon are reclaimed — never resident).
|
||||
this.windowLo = Math.max(this.horizonGen + 1, this.counter - this.recentWindowGenerations + 1)
|
||||
// reservedGensAsc (committed ∪ pending) is ascending, so chains accrue in
|
||||
// order and un-flushed single-ops in-window are indexed too.
|
||||
for (const g of this.reservedGensAsc()) {
|
||||
if (g < this.windowLo) continue
|
||||
const delta = await this.getDelta(g)
|
||||
for (const nounId of delta.nouns) appendToChain(this.nounChains, nounId, g)
|
||||
for (const verbId of delta.verbs) appendToChain(this.verbChains, verbId, g)
|
||||
const nouns = [...delta.nouns]
|
||||
const verbs = [...delta.verbs]
|
||||
for (const id of nouns) appendToChain(this.recentNounChains, id, g)
|
||||
for (const id of verbs) appendToChain(this.recentVerbChains, id, g)
|
||||
this.windowNounDeltas.set(g, nouns)
|
||||
this.windowVerbDeltas.set(g, verbs)
|
||||
}
|
||||
this.chainsReady = true
|
||||
this.chainsBuilding = null
|
||||
this.windowReady = true
|
||||
this.windowBuilding = null
|
||||
})
|
||||
}
|
||||
await this.chainsBuilding
|
||||
await this.windowBuilding
|
||||
}
|
||||
|
||||
/** Record that generation `gen` (the newest) touched these ids — keeps chains
|
||||
* current after a commit without a full rebuild. No-op until chains are built
|
||||
* (the eventual build reads `gen` from {@link committedRanges}). */
|
||||
/**
|
||||
* @description Reconstruct the FULL per-id chain for a deep pin (`gen <
|
||||
* windowLo`) — every generation that touched `id`, ascending. Built from the
|
||||
* persisted deltas BELOW the window plus the resident in-window tail, then
|
||||
* cached in the cold LRU by the caller.
|
||||
*
|
||||
* LOCK-LIGHT — deliberately does NOT hold the commit mutex (a deep read must
|
||||
* never stall writers). Correctness without the lock:
|
||||
* - The recent tail and `windowLo` are snapshotted SYNCHRONOUSLY up front, so
|
||||
* even if the window slides during the (awaited) cold scan, the chain is the
|
||||
* one consistent with the snapshot. New writes during the scan land ABOVE the
|
||||
* snapshot and cannot change any deep pin's first-after answer; the cold
|
||||
* entry is also dropped by {@link extendChains} the moment `id` is next
|
||||
* touched, so it is never served stale.
|
||||
* - The cold scan iterates a SNAPSHOT of the small committed-range list
|
||||
* (O(gaps) memory, immune to a concurrent compaction splice).
|
||||
* - A generation concurrently reclaimed by compaction surfaces as a missing
|
||||
* delta. Such a gen is always `≤ minPinned` (compaction never reclaims above
|
||||
* the lowest live pin) and therefore NEVER a live pin's answer (`first-after
|
||||
* > pin ≥ minPinned`), so it is skipped, not raised. A missing delta ABOVE
|
||||
* `minPinned` is genuine corruption and still throws.
|
||||
*
|
||||
* @param kind - `'noun'` for entities, `'verb'` for relationships.
|
||||
* @param id - The id whose chain to reconstruct.
|
||||
* @returns The id's full ascending generation chain (`[]` when never touched).
|
||||
*/
|
||||
private async reconstructColdChain(kind: 'noun' | 'verb', id: string): Promise<number[]> {
|
||||
// Synchronous, consistent snapshot of the window state.
|
||||
const windowLo = this.windowLo
|
||||
const recentTail = (kind === 'noun' ? this.recentNounChains : this.recentVerbChains).get(id)
|
||||
const recentSnapshot = recentTail ? [...recentTail] : []
|
||||
const ranges = this.committedRanges.map(([s, e]) => [s, e] as [number, number])
|
||||
const pendingCold = this.pendingGens.filter((g) => g < windowLo)
|
||||
|
||||
const cold: number[] = []
|
||||
const scan = async (g: number): Promise<void> => {
|
||||
let delta: { nouns: Set<string>; verbs: Set<string> }
|
||||
try {
|
||||
delta = await this.getDelta(g)
|
||||
} catch (err) {
|
||||
// Concurrent-compaction tolerance: a reclaimed gen (≤ minPinned, never a
|
||||
// live pin's answer) is skipped; a missing gen above the lowest pin is
|
||||
// real corruption and rethrows.
|
||||
if (g <= this.minPinnedGeneration()) return
|
||||
throw err
|
||||
}
|
||||
if ((kind === 'noun' ? delta.nouns : delta.verbs).has(id)) cold.push(g)
|
||||
}
|
||||
|
||||
for (const [s, e] of ranges) {
|
||||
if (s >= windowLo) break // ascending → nothing newer is below the window
|
||||
const hi = Math.min(e, windowLo - 1)
|
||||
for (let g = s; g <= hi; g++) await scan(g)
|
||||
}
|
||||
// Cold pending gens (only when more than W writes are buffered, i.e. tests
|
||||
// with a small window) sit between the committed gens and the window.
|
||||
for (const g of pendingCold) await scan(g)
|
||||
|
||||
// Cold gens (< windowLo) then the resident in-window tail (≥ windowLo) — both
|
||||
// ascending and disjoint, so the concatenation is the full ascending chain.
|
||||
return recentSnapshot.length ? cold.concat(recentSnapshot) : cold
|
||||
}
|
||||
|
||||
/**
|
||||
* Record that generation `gen` (the newest) touched these ids and advance the
|
||||
* hot-tail window. Keeps the resident window current after a commit without a
|
||||
* rebuild. No-op until the window is built (the eventual build reads `gen` from
|
||||
* {@link reservedGensAsc}).
|
||||
*
|
||||
* Three steps, all in-memory and I/O-FREE:
|
||||
* 1. Append `gen` to each touched id's recent chain, and DROP each touched id
|
||||
* from the cold LRU — a freshly-touched id's cached deep chain is now
|
||||
* incomplete, so it must be reconstructed on the next deep read (else a
|
||||
* deep pin whose first-after just became `gen` would wrongly read `current`).
|
||||
* 2. Record the window inverse index entry `gen → touched ids` (the slide-trim
|
||||
* driver).
|
||||
* 3. Slide `windowLo` forward to `max(horizon+1, gen-W+1)`, front-trimming the
|
||||
* chains of exactly the ids leaving the window using the inverse index — NO
|
||||
* `getDelta`, so the write path never reads `tx.json` on a slide regardless
|
||||
* of W vs the delta-cache size.
|
||||
*/
|
||||
private extendChains(gen: number, nouns: Iterable<string>, verbs: Iterable<string>): void {
|
||||
if (!this.chainsReady) return
|
||||
for (const nounId of nouns) appendToChain(this.nounChains, nounId, gen)
|
||||
for (const verbId of verbs) appendToChain(this.verbChains, verbId, gen)
|
||||
if (!this.windowReady) return
|
||||
const nounArr = [...nouns]
|
||||
const verbArr = [...verbs]
|
||||
for (const id of nounArr) {
|
||||
appendToChain(this.recentNounChains, id, gen)
|
||||
this.coldNounChains.delete(id)
|
||||
}
|
||||
for (const id of verbArr) {
|
||||
appendToChain(this.recentVerbChains, id, gen)
|
||||
this.coldVerbChains.delete(id)
|
||||
}
|
||||
this.windowNounDeltas.set(gen, nounArr)
|
||||
this.windowVerbDeltas.set(gen, verbArr)
|
||||
|
||||
const newLo = Math.max(this.horizonGen + 1, gen - this.recentWindowGenerations + 1)
|
||||
if (newLo <= this.windowLo) return
|
||||
// Collect the ids leaving the window from the inverse index (O(slid·d̄)), then
|
||||
// front-trim their chains to the new low watermark (each id once).
|
||||
const trimNouns = new Set<string>()
|
||||
const trimVerbs = new Set<string>()
|
||||
for (let g = this.windowLo; g < newLo; g++) {
|
||||
const n = this.windowNounDeltas.get(g)
|
||||
if (n) for (const id of n) trimNouns.add(id)
|
||||
this.windowNounDeltas.delete(g)
|
||||
const v = this.windowVerbDeltas.get(g)
|
||||
if (v) for (const id of v) trimVerbs.add(id)
|
||||
this.windowVerbDeltas.delete(g)
|
||||
}
|
||||
for (const id of trimNouns) dropChainFront(this.recentNounChains, id, newLo)
|
||||
for (const id of trimVerbs) dropChainFront(this.recentVerbChains, id, newLo)
|
||||
this.windowLo = newLo
|
||||
}
|
||||
|
||||
/** Drop the chains so the next read rebuilds them from the surviving
|
||||
* generations (called after compaction removes record-sets). */
|
||||
/** Drop the resident window + cold LRU so the next read rebuilds from the
|
||||
* surviving generations (called after compaction removes record-sets / on
|
||||
* reopen). */
|
||||
private invalidateChains(): void {
|
||||
this.chainsReady = false
|
||||
this.chainsBuilding = null
|
||||
this.nounChains.clear()
|
||||
this.verbChains.clear()
|
||||
this.windowReady = false
|
||||
this.windowBuilding = null
|
||||
this.recentNounChains.clear()
|
||||
this.recentVerbChains.clear()
|
||||
this.windowNounDeltas.clear()
|
||||
this.windowVerbDeltas.clear()
|
||||
this.coldNounChains.clear()
|
||||
this.coldVerbChains.clear()
|
||||
this.windowLo = 0
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1603,6 +1894,21 @@ function appendToChain(chains: Map<string, number[]>, id: string, gen: number):
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Drop the leading entries of an id's ascending chain that fall
|
||||
* below `lo` (generations that slid out of the hot-tail window), deleting the
|
||||
* chain entirely when it empties. Used by the O(d̄) window slide in
|
||||
* {@link GenerationStore.extendChains}. Tolerates an already-trimmed front.
|
||||
*/
|
||||
function dropChainFront(chains: Map<string, number[]>, id: string, lo: number): void {
|
||||
const chain = chains.get(id)
|
||||
if (chain === undefined) return
|
||||
let i = 0
|
||||
while (i < chain.length && chain[i] < lo) i++
|
||||
if (i === chain.length) chains.delete(id)
|
||||
else if (i > 0) chain.splice(0, i)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Binary-search an ascending generation chain for the FIRST entry
|
||||
* strictly greater than `gen` (the generation whose before-image holds the
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue