feat(log): the guarded log-authority core — group-commit durable-at-ack, the per-brain switch, the verification oracle
The storage-authority adoption path, guarded shape: the canonical tree
stays authoritative by default ('tree'); a brain flips to 'log' only
through the verification oracle, and the flip is stored, per-brain,
checked at open only.
- FactLog.ensureSynced(): classic group commit — concurrent writers
append, then join ONE covering fsync (running + queued slots give the
covering guarantee: the sync a caller awaits always starts after its
append landed). Solo writer = immediate sync.
- GenerationStore.logDurability 'deferred' (default, byte-identical to
today: fact durability rides the group-commit flush, ack latency
unchanged) | 'at-ack' (log-authority mode: every single-op ack awaits a
covering log fsync — an acked write's fact survives power loss, by
contract). transact() was already durable-at-return in both modes.
- src/db/logAuthority.ts: the stored switch artifact
(_system/log-authority.json, absent = tree), readLogAuthority, and the
VERIFICATION ORACLE — replay the fact log, fold latest state per id
(digests, never bodies — memory-bounded), diff against the canonical
tree paged; verdict green iff every canonical row is exactly reproduced
AND the log claims nothing canonical denies. Divergences are NAMED by
class (pre-log-record → needs baseline backfill; state-differs;
log-live-canonical-absent; log-tombstone-canonical-present). The flip
REFUSES on red with the first divergence and the cure in the message.
- Brainy: authority read at open (log → durable-at-ack enabled);
logAuthority() / verifyLogAuthority() / adoptLogAuthority() public API.
Nothing flips by itself; nothing changes for existing brains.
This commit is contained in:
parent
9fda6d9566
commit
6595309765
4 changed files with 409 additions and 3 deletions
|
|
@ -194,6 +194,15 @@ import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js
|
|||
import { GenerationConflictError, StoreInconsistentError } from './db/errors.js'
|
||||
import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js'
|
||||
import { assessIndexReadiness } from './utils/indexReadiness.js'
|
||||
import {
|
||||
readLogAuthority,
|
||||
runLogCompletenessOracle,
|
||||
flipToLogAuthority,
|
||||
recordDigest,
|
||||
type LogAuthorityRecord,
|
||||
type LogAuthorityStorage,
|
||||
type OracleReport
|
||||
} from './db/logAuthority.js'
|
||||
import { MemoryStorage } from './storage/adapters/memoryStorage.js'
|
||||
import type {
|
||||
CompactHistoryOptions,
|
||||
|
|
@ -701,6 +710,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// background worker. A crash can delay a vector, never lose one.
|
||||
private _pendingEmbedIds = new Set<string>()
|
||||
private _embedWorkerFlight: Promise<void> | null = null
|
||||
|
||||
/** The stored log-authority switch, read once at open (default: tree). */
|
||||
private _logAuthority: LogAuthorityRecord = { authority: 'tree' }
|
||||
// A failed walk latches its error: retries within the cooldown rethrow it
|
||||
// instantly instead of re-walking, so a tight caller-side retry loop costs
|
||||
// one loud error per query, never a full store walk per query.
|
||||
|
|
@ -1424,6 +1436,19 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this._generationStampingActive = true
|
||||
}
|
||||
|
||||
// LOG-AUTHORITY SWITCH (checked at open only): a brain that has
|
||||
// flipped to log-authoritative storage gets durable-at-ack fact
|
||||
// writes (group-committed fsync covering every ack). Default 'tree'
|
||||
// = today's behavior, zero added latency.
|
||||
if (!this.isReadOnly) {
|
||||
const authority = await readLogAuthority(this.storage)
|
||||
this._logAuthority = authority
|
||||
if (authority.authority === 'log') {
|
||||
this.generationStore.setLogDurability('at-ack')
|
||||
prodLog.info('[Brainy] storage authority: generation log (durable-at-ack enabled)')
|
||||
}
|
||||
}
|
||||
|
||||
// MT5 crash recovery: reload the durable pending-embed markers (a
|
||||
// BOUNDED prefix listing — never a store walk) and resume the worker
|
||||
// in the background. A crash between a deferred write's ack and its
|
||||
|
|
@ -7721,6 +7746,62 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return this.generationStore?.getFactLog()?.segmentPaths(options) ?? []
|
||||
}
|
||||
|
||||
/**
|
||||
* @description This brain's storage authority as read at open: `'tree'`
|
||||
* (the canonical record tree is authoritative; the generation log is a
|
||||
* complete dual-written journal — the default) or `'log'` (the log is
|
||||
* authoritative; single-op acks are durable-at-ack). See
|
||||
* {@link adoptLogAuthority} for the guarded flip.
|
||||
*/
|
||||
logAuthority(): LogAuthorityRecord {
|
||||
return { ...this._logAuthority }
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Run the log-completeness VERIFICATION ORACLE (read-only):
|
||||
* replay the generation log and diff the resulting per-id state against
|
||||
* the canonical tree. Green = the log exactly reproduces canonical truth.
|
||||
* Red NAMES every divergence class — `pre-log-record` rows (canonical
|
||||
* history the log never saw) need a baseline backfill before this brain
|
||||
* can ever flip. Safe at any time; walks are paged and memory-bounded
|
||||
* (digests, never bodies).
|
||||
*/
|
||||
async verifyLogAuthority(): Promise<OracleReport> {
|
||||
await this.ensureInitialized()
|
||||
return runLogCompletenessOracle({
|
||||
storage: this.storage as unknown as LogAuthorityStorage,
|
||||
scanFacts: () => this.scanFacts(),
|
||||
canonicalNounDigest: async (id: string) => {
|
||||
const raw = await this.storage.readNounRaw(id)
|
||||
if (raw.metadata === null && raw.vector === null) return null
|
||||
return recordDigest({ metadata: raw.metadata, vector: raw.vector })
|
||||
},
|
||||
factRecordDigest: (record: unknown) => recordDigest(record)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* @description THE GUARDED FLIP: run the oracle; on GREEN, persist the
|
||||
* authority switch and enable durable-at-ack immediately (the rest of
|
||||
* log-authoritative behavior engages at the next open — the switch is
|
||||
* checked-at-open by law). On RED the flip REFUSES, naming the first
|
||||
* divergence and the cure. One-directional unless an operator reverts
|
||||
* the stored artifact explicitly.
|
||||
* @returns The oracle report (green) — callers surface it as the flip receipt.
|
||||
* @throws When the oracle is red; nothing is written.
|
||||
*/
|
||||
async adoptLogAuthority(): Promise<OracleReport> {
|
||||
await this.ensureInitialized()
|
||||
this.assertWritable('adoptLogAuthority')
|
||||
const report = await this.verifyLogAuthority()
|
||||
this._logAuthority = await flipToLogAuthority(
|
||||
this.storage as unknown as LogAuthorityStorage,
|
||||
report
|
||||
)
|
||||
this.generationStore.setLogDurability('at-ack')
|
||||
return report
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the reified transaction log — one entry per committed
|
||||
* generation, carrying the committed generation, the commit timestamp, and
|
||||
|
|
|
|||
Reference in a new issue