Compare commits
9 commits
reclaim/pa
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 4f1e27c9a0 | |||
| 297a3d7657 | |||
| 7ab670b525 | |||
| f097cbf6f2 | |||
| e64e2bc175 | |||
| 655aa13ea7 | |||
| 39c71ecdac | |||
| 0759c03a82 | |||
|
|
298cb6daca |
6 changed files with 433 additions and 27 deletions
76
releases/brainy.json
Normal file
76
releases/brainy.json
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
{
|
||||
"product": "brainy",
|
||||
"entries": [
|
||||
{
|
||||
"version": "11.0.5",
|
||||
"date": "2026-09-02",
|
||||
"headline": "Graph-first finds in production, and opens that stop rescanning history",
|
||||
"items": [
|
||||
"find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.",
|
||||
"related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).",
|
||||
"Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.4",
|
||||
"date": "2026-09-01",
|
||||
"headline": "Closes in milliseconds, index rebuilds without the disk-sync storm",
|
||||
"items": [
|
||||
"close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.",
|
||||
"The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.",
|
||||
"A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.3",
|
||||
"date": "2026-09-01",
|
||||
"headline": "The embedding upgrade ceremony runs on every brain",
|
||||
"items": [
|
||||
"A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.",
|
||||
"A one-fix release; nothing else changed."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.2",
|
||||
"date": "2026-08-31",
|
||||
"headline": "One embedding quality everywhere, 3–4× faster imports",
|
||||
"items": [
|
||||
"Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.",
|
||||
"Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.",
|
||||
"The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.1",
|
||||
"date": "2026-08-31",
|
||||
"headline": "Deletes inside transactions are safe",
|
||||
"items": [
|
||||
"Deleting relations inside a transact() no longer corrupts index bookkeeping.",
|
||||
"A store that deletes its last relation keeps serving instead of refusing."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "11.0.0",
|
||||
"date": "2026-08-28",
|
||||
"headline": "One install, one engine — Brainy",
|
||||
"items": [
|
||||
"The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.",
|
||||
"A missing native build refuses loudly with its cures named; nothing falls back silently.",
|
||||
"Stores open in place — no migration."
|
||||
],
|
||||
"url": null,
|
||||
"thumb": null
|
||||
}
|
||||
],
|
||||
"history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md."
|
||||
}
|
||||
110
releases/open-brainy.json
Normal file
110
releases/open-brainy.json
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
{
|
||||
"product": "open-brainy",
|
||||
"entries": [
|
||||
{
|
||||
"version": "10.4.9",
|
||||
"date": "2026-09-02",
|
||||
"headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history",
|
||||
"items": [
|
||||
"find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).",
|
||||
"related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.",
|
||||
"Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.7",
|
||||
"date": "2026-09-01",
|
||||
"headline": "Count ledgers can no longer race themselves",
|
||||
"items": [
|
||||
"Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.",
|
||||
"Atomic writes carry a per-process sequence, so two processes' temp files can never collide."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.6",
|
||||
"date": "2026-08-31",
|
||||
"headline": "Transactions cross the index seam safely",
|
||||
"items": [
|
||||
"Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.",
|
||||
"Fixes a class of transaction failures on stores with integer-mapped relation endpoints."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.5",
|
||||
"date": "2026-08-31",
|
||||
"headline": "Recovery tells the truth, docs live at home",
|
||||
"items": [
|
||||
"A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.",
|
||||
"A sealed segment declares only the generations it actually holds.",
|
||||
"The engine's documentation now publishes from its own repository."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.4",
|
||||
"date": "2026-08-28",
|
||||
"headline": "Faster opens, quieter idle",
|
||||
"items": [
|
||||
"Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.",
|
||||
"The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.",
|
||||
"A slow open now names the exact step it is in, so operators see what is being paid and why."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.3",
|
||||
"date": "2026-08-27",
|
||||
"headline": "Open Brainy, under its own name",
|
||||
"items": [
|
||||
"The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.",
|
||||
"No code changes; your imports change once and everything else stays put."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.2",
|
||||
"date": "2026-08-27",
|
||||
"headline": "Vectors that lie are refused, counts that drift are caught",
|
||||
"items": [
|
||||
"A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.",
|
||||
"The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.",
|
||||
"Plugin activation failures keep their original error as cause, so the real frame reaches your logs."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.1",
|
||||
"date": "2026-08-26",
|
||||
"headline": "Writes that change nothing cost nothing",
|
||||
"items": [
|
||||
"The read gate is per index family, and a write carrying unchanged data never re-embeds.",
|
||||
"The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1",
|
||||
"thumb": null
|
||||
},
|
||||
{
|
||||
"version": "10.4.0",
|
||||
"date": "2026-08-26",
|
||||
"headline": "Repair routing, the vector ledger, and honest empties",
|
||||
"items": [
|
||||
"Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.",
|
||||
"An empty string is real data, not a missing field.",
|
||||
"The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed."
|
||||
],
|
||||
"url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0",
|
||||
"thumb": null
|
||||
}
|
||||
],
|
||||
"history": "Earlier releases are recorded in CHANGELOG.md in this repository."
|
||||
}
|
||||
|
|
@ -248,17 +248,12 @@ else
|
|||
echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n"
|
||||
fi
|
||||
|
||||
# Step 12: Push public docs to the soulcraft.com docs ingest door
|
||||
# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when
|
||||
# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish —
|
||||
# that already happened) when a push errors, so the docs site never
|
||||
# silently trails npm.
|
||||
echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}"
|
||||
if node scripts/push-docs.js; then
|
||||
echo -e "${GREEN}✅ Docs push step done${NC}\n"
|
||||
else
|
||||
echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n"
|
||||
fi
|
||||
# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled):
|
||||
# soulcraft.com/docs carries the paid product's documentation only. This
|
||||
# engine's documentation home is THIS repository — README and docs/ — and the
|
||||
# site serves 301s for the slugs this rail used to push. The push script stays
|
||||
# in the tree for history; the rail no longer calls it.
|
||||
echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}"
|
||||
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}"
|
||||
|
|
|
|||
121
src/brainy.ts
121
src/brainy.ts
|
|
@ -12497,6 +12497,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* healed by `repairIndex()`, whose unconditional recount rebuilds the
|
||||
* rollups from a canonical walk and re-stamps. Best-effort: a stamp-write
|
||||
* fault warns loudly but never fails the flush that carried real data.
|
||||
*
|
||||
* THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is
|
||||
* the ALLOCATED counter — a number a write in flight has claimed and may
|
||||
* never commit. Stamping it made the stamp's generation label a claim about
|
||||
* counts it was not taken at, and every crash inside a write window then
|
||||
* produced a spurious verdict at the next open: either `sourceGeneration N
|
||||
* is ahead of the log head N-1` (the allocated generation died with the
|
||||
* process) or `rollup invariant 'nounCount': stamped X, observed Y` (the
|
||||
* recovery fold folded facts the stamp's counts predate). MEASURED on the
|
||||
* crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on
|
||||
* a coherent store raised one of those two verdicts, each of them naming
|
||||
* `repairIndex()` — a whole-store recount — as the cure for nothing.
|
||||
*/
|
||||
private async stampEntityTree(): Promise<void> {
|
||||
if (this.isReadOnly) return
|
||||
|
|
@ -12507,7 +12519,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
])
|
||||
await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, {
|
||||
family: 'entity-tree',
|
||||
sourceGeneration: this.generationStore.generation(),
|
||||
sourceGeneration: this.generationStore.committedGeneration(),
|
||||
members: { mode: 'rollup', invariants: { nounCount, verbCount } }
|
||||
})
|
||||
} catch (error) {
|
||||
|
|
@ -12520,16 +12532,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
/**
|
||||
* @description Open-time coherence check for the entity tree's family stamp:
|
||||
* compare `sourceGeneration` against the log head and the stamped rollup
|
||||
* invariants against the live counters. Verdicts:
|
||||
* compare `sourceGeneration` against the store's COMMITTED generation and
|
||||
* the stamped rollup invariants against the live counters. Verdicts:
|
||||
* - `coherent` / `absent` (legacy store; first flush stamps) → silent.
|
||||
* - `behind` → benign for the tree (it is written BY the commit; only the
|
||||
* stamp is stale — a crash landed between commit and flush). Refreshed at
|
||||
* the next flush.
|
||||
* - `torn` → a TORN GENERATION-LOG TAIL, handled by
|
||||
* {@link demoteTornEntityTreeStamp}: terminal, never a wait.
|
||||
* - `incoherent` → LOUD: the tree or its counters diverged from what was
|
||||
* stamped — `repairIndex()` recounts from canonical and re-stamps.
|
||||
* Never blocks open; a fault reading the stamp is surfaced as unverifiable,
|
||||
* never conflated with absence.
|
||||
*
|
||||
* THE COMPARISON IS AGAINST `committedGeneration()`, matching what
|
||||
* {@link stampEntityTree} writes and what every other open-time watermark in
|
||||
* this class already reasons about (the fact-scan capability, the metadata /
|
||||
* graph / HNSW watermark verdicts). Comparing against the allocated counter
|
||||
* was the one place that disagreed, and disagreeing was the whole defect.
|
||||
*/
|
||||
private async verifyEntityTreeStamp(): Promise<void> {
|
||||
let stamp: FamilyStamp | null
|
||||
|
|
@ -12546,11 +12566,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this.storage.getNounCount(),
|
||||
this.storage.getVerbCount()
|
||||
])
|
||||
const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), {
|
||||
const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), {
|
||||
nounCount,
|
||||
verbCount
|
||||
})
|
||||
if (verdict.state === 'incoherent') {
|
||||
if (verdict.state === 'torn') {
|
||||
await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, {
|
||||
nounCount,
|
||||
verbCount
|
||||
})
|
||||
} else if (verdict.state === 'incoherent') {
|
||||
prodLog.warn(
|
||||
`[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` +
|
||||
`The canonical tree or its counters diverged from the stamped state — run ` +
|
||||
|
|
@ -12564,6 +12589,92 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description THE TERMINAL VERDICT for a torn generation-log tail.
|
||||
*
|
||||
* A stamp whose `sourceGeneration` sits ABOVE the store's committed
|
||||
* watermark witnesses a generation that is not in the log: the stamp's fsync
|
||||
* outlived the tail's. By the time this runs, log-authority recovery has
|
||||
* already folded every intact fact above the manifest and advanced the
|
||||
* watermark to cover them — so if the stamp is STILL ahead, the generation
|
||||
* it names is not merely late, it is GONE. There is nothing to wait for.
|
||||
*
|
||||
* That is the whole point of this method. A field report of this class
|
||||
* (single-process store, abrupt termination mid-fold) described a reopen
|
||||
* that narrated the tear and then held 100% CPU with zero log growth for
|
||||
* eight minutes before an operator wiped the directory. A recovery that
|
||||
* cannot say what it is waiting for has no business spinning; the honest
|
||||
* answer here is a verdict, taken now, at O(1) cost.
|
||||
*
|
||||
* WHAT THE VERDICT DOES — the stamped surface is UNUSABLE, so it is
|
||||
* discarded rather than believed: the stamped counts describe a generation
|
||||
* that never became durable, and comparing them against live counters can
|
||||
* only produce noise. The tree itself is not in question (it IS canonical —
|
||||
* every commit writes it, and the fold re-applied every after-image the log
|
||||
* still holds), so the demotion is a re-derivation of this family's verified
|
||||
* surface at the generation the store can actually show:
|
||||
*
|
||||
* - WRITER open → re-stamp at `committedGeneration()` from the live
|
||||
* counters — exactly what the next flush would write, taken now so the
|
||||
* tear cannot re-narrate on every subsequent open. Both count sets are
|
||||
* logged so an operator can see whether anything really moved.
|
||||
* - READER open → a reader cannot re-stamp. Narrate the same terminal
|
||||
* verdict with the named cure and carry on serving; a read-only inspector
|
||||
* is never locked out of a store, and never left waiting either.
|
||||
*
|
||||
* BOUNDEDNESS: straight-line code. No loop, no retry, no await on any
|
||||
* external progress signal — the two counter reads and one stamp write are
|
||||
* the entire cost, and none of them scales with the store.
|
||||
*/
|
||||
private async demoteTornEntityTreeStamp(
|
||||
stamp: FamilyStamp,
|
||||
stampSource: number,
|
||||
head: number,
|
||||
observed: { nounCount: number; verbCount: number }
|
||||
): Promise<void> {
|
||||
const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {}
|
||||
const detail =
|
||||
`[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` +
|
||||
`generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` +
|
||||
`generation is ${head} after crash recovery — the stamp's fsync outlived the log tail's, ` +
|
||||
`and generation ${stampSource} is not in the log to arrive. Stamped rollups ` +
|
||||
`${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.`
|
||||
|
||||
if (this.isReadOnly) {
|
||||
prodLog.warn(
|
||||
`${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` +
|
||||
`family stays UNVERIFIED for this session (reads are unaffected — the canonical tree ` +
|
||||
`is the truth this stamp only describes). Cure: open the store with a writer, or run ` +
|
||||
`brain.repairIndex() there, to recount from canonical and re-stamp.`
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
const startedAt = Date.now()
|
||||
try {
|
||||
await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, {
|
||||
family: 'entity-tree',
|
||||
sourceGeneration: head,
|
||||
members: {
|
||||
mode: 'rollup',
|
||||
invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount }
|
||||
}
|
||||
})
|
||||
prodLog.warn(
|
||||
`${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` +
|
||||
`from the live counters in ${Date.now() - startedAt}ms — terminal, not a wait. If the ` +
|
||||
`observed counts above look wrong for your data, run brain.repairIndex() to recount ` +
|
||||
`from canonical.`
|
||||
)
|
||||
} catch (error) {
|
||||
prodLog.warn(
|
||||
`${detail} The demotion's re-stamp FAILED (${(error as Error).message}) — the tear will ` +
|
||||
`narrate again at the next open, which is the honest outcome; the store still serves ` +
|
||||
`from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the writer process serving this data directory to flush its in-memory
|
||||
* indexes to disk, so a read-only inspector can observe fresh state.
|
||||
|
|
|
|||
|
|
@ -12,9 +12,11 @@
|
|||
* the verified surface is a small set of rollup invariants (entity/
|
||||
* relationship counts) plus `sourceGeneration`.
|
||||
*
|
||||
* `sourceGeneration` is the generation of the source-of-truth log this
|
||||
* projection reflects — open-time coherence becomes a COMPARISON (stamp vs
|
||||
* log head), not a walk:
|
||||
* `sourceGeneration` is the COMMITTED generation of the source-of-truth log
|
||||
* this projection reflects — never the allocated counter, which names a
|
||||
* generation that may never commit (see {@link StampVerdict.torn}) — so
|
||||
* open-time coherence becomes a COMPARISON (stamp vs committed head), not a
|
||||
* walk:
|
||||
*
|
||||
* - equal + invariants hold → coherent, serve.
|
||||
* - behind → the projection missed the tail (crash between commit and stamp);
|
||||
|
|
@ -24,6 +26,9 @@
|
|||
* - invariants FAIL at equal generation → genuine incoherence: loud, and the
|
||||
* repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a
|
||||
* canonical walk) heals it.
|
||||
* - AHEAD → a torn generation-log tail: the stamp's fsync outlived the log
|
||||
* tail's. TERMINAL, never a wait — the generation the stamp names does not
|
||||
* exist to arrive.
|
||||
*
|
||||
* Stamps are JSON on purpose — every incident gets debugged by reading a
|
||||
* stamp in a terminal.
|
||||
|
|
@ -70,6 +75,12 @@ export type StampVerdict =
|
|||
| { state: 'coherent' }
|
||||
| { state: 'absent' } // legacy store — first stamp writes at the next flush
|
||||
| { state: 'behind'; stampSource: number; head: number }
|
||||
/**
|
||||
* TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the
|
||||
* store's committed watermark can no longer show. TERMINAL — there is no
|
||||
* generation to wait for, so the open demotes (or refuses) and never spins.
|
||||
*/
|
||||
| { state: 'torn'; stampSource: number; head: number }
|
||||
| { state: 'incoherent'; failures: string[] }
|
||||
| { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence
|
||||
|
||||
|
|
@ -118,12 +129,15 @@ export function verifyFamilyStamp(
|
|||
): StampVerdict {
|
||||
if (stamp === null) return { state: 'absent' }
|
||||
if (stamp.sourceGeneration > head) {
|
||||
// A stamp AHEAD of the log claims state that never committed — the
|
||||
// projection was stamped against truth that a crash rolled back.
|
||||
return {
|
||||
state: 'incoherent',
|
||||
failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`]
|
||||
}
|
||||
// A stamp AHEAD of committed truth witnesses a generation the store can no
|
||||
// longer show: the stamp's fsync survived a crash that the log tail did
|
||||
// not. This is the TORN GENERATION-LOG TAIL — its own class, never folded
|
||||
// in with `incoherent` (a count that drifted at a generation both sides
|
||||
// agree on), because the two have opposite cures: incoherence is recounted,
|
||||
// a tear is DEMOTED. It is also terminal by construction — there is no
|
||||
// generation the open can wait for, because the one the stamp names is
|
||||
// gone.
|
||||
return { state: 'torn', stampSource: stamp.sourceGeneration, head }
|
||||
}
|
||||
if (stamp.sourceGeneration < head) {
|
||||
return { state: 'behind', stampSource: stamp.sourceGeneration, head }
|
||||
|
|
|
|||
|
|
@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => {
|
|||
const invariants = (stamp.members as any).invariants
|
||||
expect(invariants.nounCount).toBe(await brain.storage.getNounCount())
|
||||
expect(invariants.verbCount).toBe(await brain.storage.getVerbCount())
|
||||
expect(stamp.sourceGeneration).toBe(brain.generation())
|
||||
// THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the
|
||||
// counter labelled the stamp with a generation a write in flight had merely
|
||||
// claimed, so every crash inside a write window produced a spurious verdict
|
||||
// at the next open (see the torn-tail pins below).
|
||||
expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration())
|
||||
expect(stamp.generation).toBeGreaterThanOrEqual(1)
|
||||
})
|
||||
|
||||
|
|
@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => {
|
|||
expect(stillIncoherent).toEqual([])
|
||||
})
|
||||
|
||||
/**
|
||||
* Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's
|
||||
* committed watermark — the durable shape a torn generation-log tail leaves
|
||||
* behind (the stamp's fsync outlived the tail's). Fabricated rather than
|
||||
* crash-produced so the pin is deterministic; the seeded-SIGKILL lane
|
||||
* (`scripts/crash-consistency.mjs` in the engine repo) produces the same
|
||||
* shape from a real abrupt termination.
|
||||
*/
|
||||
const fabricateTear = (ahead: number): FamilyStamp => {
|
||||
const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)
|
||||
const zlib = require('node:zlib')
|
||||
const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp
|
||||
const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead }
|
||||
fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn)))
|
||||
return torn
|
||||
}
|
||||
|
||||
it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => {
|
||||
for (let i = 0; i < 3; i++)
|
||||
await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } })
|
||||
await brain.close()
|
||||
const torn = fabricateTear(5)
|
||||
|
||||
const warn = vi.spyOn(prodLog, 'warn')
|
||||
const startedAt = Date.now()
|
||||
brain = await open()
|
||||
const openMs = Date.now() - startedAt
|
||||
|
||||
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
|
||||
expect(tearLines.length).toBe(1)
|
||||
const said = String(tearLines[0][0])
|
||||
// Narrated PRECISELY: both generations, the file, and the named cure.
|
||||
expect(said).toContain(`source generation ${torn.sourceGeneration}`)
|
||||
expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`)
|
||||
expect(said).toContain(ENTITY_TREE_STAMP_PATH)
|
||||
expect(said).toContain('DEMOTED')
|
||||
expect(said).toMatch(/repairIndex\(\)/)
|
||||
// Terminal, not a wait: the demotion is O(1) straight-line work, so a tear
|
||||
// cannot turn an open into the 8-minute spin this class was reported as.
|
||||
expect(openMs).toBeLessThan(30_000)
|
||||
|
||||
// The store SERVES — a tear in a stamp never locks an owner out of the
|
||||
// canonical tree the stamp merely describes.
|
||||
expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3)
|
||||
|
||||
// The demotion CONVERGED: the stamp now names committed truth, and the
|
||||
// next open is quiet. A verdict that re-narrates every open is a wait
|
||||
// wearing a different hat.
|
||||
const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp
|
||||
expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration())
|
||||
await brain.close()
|
||||
const warn2 = vi.spyOn(prodLog, 'warn')
|
||||
brain = await open()
|
||||
expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([])
|
||||
})
|
||||
|
||||
it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => {
|
||||
await brain.add({ data: 'ro', type: 'document', metadata: {} })
|
||||
await brain.close()
|
||||
const torn = fabricateTear(3)
|
||||
|
||||
const warn = vi.spyOn(prodLog, 'warn')
|
||||
const reader: any = await Brainy.openReadOnly({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL'))
|
||||
expect(tearLines.length).toBe(1)
|
||||
const said = String(tearLines[0][0])
|
||||
expect(said).toContain('READ-ONLY')
|
||||
expect(said).toContain('UNVERIFIED')
|
||||
expect(said).toMatch(/repairIndex\(\)/)
|
||||
await reader.close()
|
||||
|
||||
// A reader never rewrites the store: read the bytes back off disk (not
|
||||
// through a writer open, which would demote them) — the torn stamp is
|
||||
// exactly as it was found.
|
||||
const onDisk = JSON.parse(
|
||||
require('node:zlib')
|
||||
.gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`)))
|
||||
.toString('utf-8')
|
||||
) as FamilyStamp
|
||||
expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration)
|
||||
expect(onDisk.generation).toBe(torn.generation)
|
||||
|
||||
brain = await open()
|
||||
})
|
||||
|
||||
it('the one verifier handles both member modes', () => {
|
||||
const rollup: FamilyStamp = {
|
||||
family: 'x',
|
||||
|
|
@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => {
|
|||
stampSource: 5,
|
||||
head: 9
|
||||
})
|
||||
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head
|
||||
// AHEAD is its own class — a torn generation-log tail, never folded in
|
||||
// with `incoherent`: the two have opposite cures (recount vs demote).
|
||||
expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({
|
||||
state: 'torn',
|
||||
stampSource: 5,
|
||||
head: 3
|
||||
})
|
||||
expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' })
|
||||
|
||||
const enumerated: FamilyStamp = {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue