fix(storage): enumeration re-keys on the identity record, not the vector leg
Some checks failed
CI / Node 22 (push) Successful in 12m19s
CI / Node 24 (push) Successful in 12m16s
CI / Integration + conformance (Node 22) (push) Failing after 14m47s
CI / Bun (latest) (push) Successful in 12m20s

The noun/verb pagination walks (getNounsWithPagination,
getNounIdsWithPagination, getVerbsWithPagination) listed shard contents by
filtering for vectors.json, while the canonical count ledger has always
counted a row by its metadata.json presence alone. A row with metadata and
no vector file was therefore counted by the ledger but never yielded by the
walk — a permanent "counted but invisible" phantom for any downstream
consumer that iterates the walk to account for the ledger's total.

Nouns now enumerate by metadata.json and hydrate the vector leg optionally,
yielding the sanctioned unvectored shape (vector: []) when it's absent.
Verbs enumerate the same way, but a metadata-only verb row can only be fully
reconstructed when sourceId/targetId happen to be recoverable from metadata
(never true for a current production write — those fields live only in the
vector leg); otherwise the row is counted but loudly skipped rather than
fabricated, since a phantom edge with fake endpoints would be worse than the
original defect.

Separately, GenerationStore's recovery-fold replay (replayFact) now applies
preserve-if-absent: a metadata-only after-image replayed over an already-
vectored row carries the existing vector forward instead of deleting it via
writeNounRaw/writeVerbRaw's exact-restore null-means-delete contract (which
must stay exact for transaction-abort rollback). A genuine tombstone still
removes both legs.
This commit is contained in:
David Snelling 2026-08-27 12:38:52 -07:00
parent 4c7b0fab7a
commit f8d8ce16b9
3 changed files with 519 additions and 25 deletions

View file

@ -654,10 +654,43 @@ export class GenerationStore {
let replayed = 0 let replayed = 0
const replayFact = async (fact: CommitFact): Promise<void> => { const replayFact = async (fact: CommitFact): Promise<void> => {
for (const op of fact.ops) { for (const op of fact.ops) {
const image = let image: { metadata: unknown | null; vector: unknown | null }
op.record === null if (op.record === null) {
? { metadata: null, vector: null } // A genuine tombstone (both legs absent) — the fold removes
: { metadata: op.record.metadata, vector: op.record.vector } // both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw
// exact-restore contract.
image = { metadata: null, vector: null }
} else if (
op.record.metadata !== null &&
(op.record.vector === null || op.record.vector === undefined)
) {
// PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's
// half): a metadata-only after-image must never DELETE an
// existing vector leg through the fold. `writeNounRaw`/
// `writeVerbRaw` are exact-restore primitives — a `vector:
// null` there means "delete", which is exactly right for
// `rollBackUncommittedGeneration`'s before-image restore (a
// transaction abort legitimately un-writes a vector the failed
// transaction added). It is NOT right here: this fold replays
// AFTER-IMAGES, and re-applying an already-intact record must
// be byte-safe (this module's own invariant, see the log-authority
// comment above) — silently erasing a landed vector because one
// replayed fact's vector leg came back null is the exact defect
// that left metadata-counted, never-enumerated rows in a
// production store (confirmed root cause: the enumeration walk
// used to key on the vector leg, so a preserved-but-then-deleted
// vector made the row invisible while the ledger still counted
// it by metadata). A genuine "unvector" has its own sanctioned,
// ledger-correct path (`Brainy.unvectorNounForRootMigration`) —
// never this raw primitive, and never the fold.
const current =
op.kind === 'verb'
? await this.storage.readVerbRaw(op.id)
: await this.storage.readNounRaw(op.id)
image = { metadata: op.record.metadata, vector: current.vector ?? null }
} else {
image = { metadata: op.record.metadata, vector: op.record.vector }
}
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
else await this.storage.writeNounRaw(op.id, image) else await this.storage.writeNounRaw(op.id, image)
this.noteCheckpointDirty(op.kind, op.id) this.noteCheckpointDirty(op.kind, op.id)

View file

@ -203,6 +203,40 @@ function idFromVectorPath(path: string): string {
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
} }
/**
* @description Extract the entity id embedded in a metadata path
* (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) the IDENTITY-RECORD
* mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their
* population on this file (ADR-008 G1: the metadata record IS the population;
* the vector leg is optional), so walk ordering and cursor resume derive the
* id from THIS path, never the vector path a row with metadata and no
* vector file must still be listed, ordered, and resumable.
* @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`).
* @returns The entity id (the path segment immediately before `/metadata.json`).
*/
function idFromMetadataPath(path: string): string {
const withoutSuffix = path.replace(/\/metadata\.json$/, '')
const lastSlash = withoutSuffix.lastIndexOf('/')
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
}
/**
* @description The sanctioned UNVECTORED shape for a noun hydrated during
* enumeration when its identity record (metadata.json) exists but its vector
* leg (vectors.json) does not a fold-born metadata-only after-image, or any
* row genuinely without a vector yet. Mirrors the shape
* `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned
* unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a
* walk-yielded unvectored row is byte-shape-identical to one produced by that
* migration. Callers already handle `vector: []` as first-class
* (validateAddParams exempts it; index gates key on `length > 0`).
* @param id - The noun id.
* @returns A structurally-valid, vector-empty `HNSWNoun`.
*/
function unvectoredNoun(id: string): HNSWNoun {
return { id, vector: [], connections: new Map<number, Set<string>>(), level: 0 }
}
/** /**
* Get ID-first path for verb metadata * Get ID-first path for verb metadata
* No type parameter needed - direct O(1) lookup by ID * No type parameter needed - direct O(1) lookup by ID
@ -1453,6 +1487,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
* rollups are derived state with their own rebuild paths * rollups are derived state with their own rebuild paths
* (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`). * (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`).
* *
* EXACT-RESTORE PRIMITIVE `vector: null` DELETES the vector leg, on
* purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on
* this to legitimately un-write a vector a failed transaction added. This
* is deliberately NOT "preserve if absent" a caller replaying an
* AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must
* apply preserve-if-absent itself BEFORE calling this, by reading the
* current vector and carrying it forward when the after-image's own
* vector leg is null/undefined but its metadata is not (see `replayFact`
* for the implementation and full rationale). A caller that genuinely
* wants to unvector a row uses the sanctioned, ledger-correct path
* (`Brainy.unvectorNounForRootMigration`) never this primitive.
*
* @param id - The entity id. * @param id - The entity id.
* @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}. * @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}.
*/ */
@ -1489,7 +1535,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
/** /**
* Restore a relationship's raw stored objects byte-for-byte (verb-side * Restore a relationship's raw stored objects byte-for-byte (verb-side
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats). * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats,
* same EXACT-RESTORE contract `vector: null` deletes, on purpose; the
* fold's preserve-if-absent logic lives at its call site, not here).
* *
* @param id - The relationship id. * @param id - The relationship id.
* @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}. * @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}.
@ -2183,9 +2231,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Stable within-shard order (by noun id) so offset windows and cursor resume // Stable within-shard order (by noun id) so offset windows and cursor resume
// are deterministic; ids come from the path so skipped nouns are never read. // are deterministic; ids come from the path so skipped nouns are never read.
//
// IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record
// (not the vector) IS the population — a noun with metadata and no vector
// file (a fold-born after-image, see writeNounRaw's preserve-if-absent
// contract) must still enumerate. Keying on metadata.json here means the
// ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed)
// and this walk agree on population by construction. Ordering is
// unaffected for a healthy store: every vectored noun has both legs, so
// the id set and sort order are identical to the old vectors.json keying.
const entries = nounFiles const entries = nounFiles
.filter((p) => p.includes('/vectors.json')) .filter((p) => p.includes('/metadata.json'))
.map((p) => ({ path: p, id: idFromVectorPath(p) })) .map((p) => ({ path: p, id: idFromMetadataPath(p) }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
@ -2210,13 +2267,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
) { ) {
const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
const hydrated = await Promise.all( const hydrated = await Promise.all(
batch.map(async ({ path: nounPath }) => { batch.map(async ({ path: metadataPath, id }) => {
try { try {
const noun = await this.readCanonicalObject(nounPath) const metadata = await this.readCanonicalObject(metadataPath)
if (!noun) return null
const deserialized = this.deserializeNoun(noun)
const metadata = await this.getNounMetadata(deserialized.id)
if (!metadata) return null if (!metadata) return null
// The vector leg is OPTIONAL (population law): a metadata-only
// row hydrates with the sanctioned unvectored shape rather than
// being dropped from the walk. A fault reading the vector leg
// is treated the same as absence — best-effort, matching the
// canonical recount's tolerance for an unreadable vectors.json
// (rebuildTypeCounts) — a vector-leg problem never hides an
// otherwise-good identity record.
let deserialized: HNSWNoun
try {
const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id))
deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id)
} catch {
deserialized = unvectoredNoun(id)
}
return { deserialized, metadata } return { deserialized, metadata }
} catch (error) { } catch (error) {
// A TORN record must surface typed — a paginated read that // A TORN record must surface typed — a paginated read that
@ -2226,7 +2294,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// walk's job is to HEAL PAST it — skip the victim, serve the rest. // walk's job is to HEAL PAST it — skip the victim, serve the rest.
// Identity point-reads (get-by-id) still throw typed upstream. // Identity point-reads (get-by-id) still throw typed upstream.
if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ }
// Skip nouns that fail to load // Skip nouns whose IDENTITY record fails to load (the metadata
// read above) — that is the one leg this walk cannot proceed
// without.
return null return null
} }
}) })
@ -2347,9 +2417,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const shardDir = `entities/nouns/${shardHex}` const shardDir = `entities/nouns/${shardHex}`
try { try {
const nounFiles = await this.listCanonicalObjects(shardDir) const nounFiles = await this.listCanonicalObjects(shardDir)
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching
// comment in getNounsWithPagination: metadata.json is the population;
// the vector leg is optional, so a metadata-only row must still be
// listed (and here, for the unfiltered case, needs ZERO reads either
// way — the id comes straight from the path).
const entries = nounFiles const entries = nounFiles
.filter((p) => p.includes('/vectors.json')) .filter((p) => p.includes('/metadata.json'))
.map((p) => idFromVectorPath(p)) .map((p) => idFromMetadataPath(p))
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
const toWalk = const toWalk =
cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
@ -2560,23 +2635,79 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Stable within-shard order (by verb id) so offset windows and cursor resume // Stable within-shard order (by verb id) so offset windows and cursor resume
// are deterministic and consistent across calls. Ids come from the path, so // are deterministic and consistent across calls. Ids come from the path, so
// verbs skipped by the cursor are never read. // verbs skipped by the cursor are never read.
//
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of
// this comment in getNounsWithPagination applies here too: metadata.json
// is the population; keying on it here means this walk and the ledger
// recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed)
// agree on population by construction. Unchanged for a healthy store —
// `relate()` always writes both legs of a verb in the same commit, so
// the id set and order match the old vectors.json keying exactly; this
// only additionally surfaces a fold-born metadata-only row (see
// writeVerbRaw's preserve-if-absent contract).
const entries = verbFiles const entries = verbFiles
.filter((p) => p.includes('/vectors.json')) .filter((p) => p.includes('/metadata.json'))
.map((p) => ({ path: p, id: idFromVectorPath(p) })) .map((p) => ({ path: p, id: idFromMetadataPath(p) }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
for (const { path: verbPath, id: verbId } of entries) { for (const { path: metadataPath, id: verbId } of entries) {
if (collected.length >= peekCount) break if (collected.length >= peekCount) break
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
// (later shards are processed in full). No read for skipped verbs. // (later shards are processed in full). No read for skipped verbs.
if (cursor && shard === cursor.shard && verbId <= cursor.id) continue if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
try { try {
const rawVerb = await this.readCanonicalObject(verbPath) // Identity leg first — required. A verb this walk cannot read
if (!rawVerb) continue // metadata for cannot be hydrated at all (same as before).
const metadata = await this.readCanonicalObject(metadataPath)
if (!metadata) continue
// Deserialize connections Map from JSON storage format // The vector leg is the verb's STRUCTURAL core (verb/sourceId/
const verb = this.deserializeVerb(rawVerb) // targetId live there — see coreTypes.ts HNSWVerb), unlike a
// noun's vector, which is pure embedding data. `relate()` always
// writes both legs atomically and verbs have no deferred-embed
// path, so a healthy store's verbs always have both. A vector-leg
// absence here can only be a fold-born after-image (see
// writeVerbRaw's preserve-if-absent contract) — and unlike a
// noun, this walk cannot safely FABRICATE sourceId/targetId to
// synthesize a structurally-valid verb (an empty-string endpoint
// would silently create a phantom edge — worse than omission).
// If the metadata record happens to carry its own sourceId/
// targetId (never true for current production writes, but not
// disallowed — e.g. a future schema or a repair tool could
// populate them), reconstruct from those; otherwise this row is
// loudly skipped — counted by the ledger, but not returned as an
// item, until a repair can supply the missing endpoints.
const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId))
let verb: HNSWVerb
if (rawVerb) {
verb = this.deserializeVerb(rawVerb)
} else {
const metaSourceId = (metadata as Record<string, unknown>).sourceId
const metaTargetId = (metadata as Record<string, unknown>).targetId
const metaVerbType = (metadata as Record<string, unknown>).verb
if (
typeof metaSourceId === 'string' && metaSourceId.length > 0 &&
typeof metaTargetId === 'string' && metaTargetId.length > 0 &&
typeof metaVerbType === 'string' && metaVerbType.length > 0
) {
verb = {
id: verbId,
vector: [],
connections: new Map<number, Set<string>>(),
verb: metaVerbType as VerbType,
sourceId: metaSourceId,
targetId: metaTargetId
}
} else {
prodLog.error(
`[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` +
`record but no vector leg and no recoverable sourceId/targetId — ` +
`skipping (counted by the ledger, not yielded; needs repair).`
)
continue
}
}
// Apply type filter // Apply type filter
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) { if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
@ -2593,9 +2724,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
continue continue
} }
// Load metadata
const metadata = await this.getVerbMetadata(verb.id)
// Apply subtype filter (requires metadata — checked AFTER load) // Apply subtype filter (requires metadata — checked AFTER load)
if (filterSubtypes) { if (filterSubtypes) {
const subtype = metadata?.subtype as string | undefined const subtype = metadata?.subtype as string | undefined

View file

@ -0,0 +1,333 @@
/**
* @module tests/integration/enumeration-population-law
* @description THE POPULATION LAW (ADR-008 G1): the unfiltered noun/verb walk
* and the canonical ALL scalar must agree on the population a row's
* IDENTITY RECORD (metadata.json) is what defines membership; the vector leg
* is optional data, never a gate on visibility. Before this fix, the walk
* (getNounsWithPagination / getNounIdsWithPagination / getVerbsWithPagination)
* enumerated by keying on the VECTOR leg (`vectors.json`), so a row with
* metadata and no vector file was counted by the ledger (already
* metadata.json-keyed see `rebuildTypeCounts`) but never yielded by the
* walk: a permanent "counted but invisible" phantom for any downstream
* consumer (a health-coverage row, an index-fill walk) that iterates the walk
* to account for the ledger's total.
*
* Two legs are pinned here:
* (a)/(b) LEG 1 the walk re-keys on metadata.json. A fold-born
* metadata-only row (the exact shape `GenerationStore.replayFact` can
* leave behind, and the exact shape `writeNounRaw`/`writeVerbRaw` accept)
* must be YIELDED, hydrated with the sanctioned unvectored shape
* (`vector: []`) not merely counted.
*
* For VERBS this closes only PARTIALLY: `sourceId`/`targetId` are
* HNSWVerb's structural core and live ONLY in the vector leg (never in
* metadata see `RESERVED_RELATION_FIELDS` in reservedFields.ts, which
* does not include them). A metadata-only verb row therefore cannot be
* safely reconstructed without FABRICATING an edge's endpoints which
* would silently create a phantom relationship, strictly worse than the
* original defect. The walk recovers the row when its metadata happens
* to carry `sourceId`/`targetId` (a defensive, forward-compatible
* fallback never true for a CURRENT production write, but not
* disallowed either); otherwise it counts the row (ledger, unchanged)
* but loudly skips yielding it, logging the gap instead of hiding it.
* Closing this fully requires persisting `sourceId`/`targetId` in verb
* metadata a schema change out of this task's scope; see the session
* report for the explicit call-out.
*
* (c)/(d) LEG 2 the recovery fold's preserve-if-absent contract, exercised
* directly against `GenerationStore`/`FactLog` (below the `Brainy` API):
* a metadata-only after-image replayed over an already-vectored row must
* PRESERVE the existing vector leg (never delete it); a genuine tombstone
* (both legs absent) still removes both legs.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { randomUUID } from 'node:crypto'
import { Brainy } from '../../src/index.js'
import { GenerationStore } from '../../src/db/generationStore.js'
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
import { LOG_AUTHORITY_PATH } from '../../src/db/logAuthority.js'
import type { CommitFact } from '../../src/db/factLog.js'
describe('enumeration population law — LEG 1 (identity-keyed walk)', () => {
let dir: string
let brain: any
const open = async () => {
const b: any = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true,
dimensions: 384
})
await b.init()
return b
}
beforeEach(async () => {
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-population-law-'))
brain = await open()
})
afterEach(async () => {
await brain.close?.().catch(() => {})
fs.rmSync(dir, { recursive: true, force: true })
})
it('(a) nouns.all equals the unfiltered walk-yield count with a fold-born metadata-only row present', async () => {
// Ordinary, fully-vectored background population.
await brain.add({ data: 'one', type: 'document' })
await brain.add({ data: 'two', type: 'document' })
await brain.flush()
// THE EXACT PRE-FIX SHAPE: a metadata-only row against a FRESH id — no
// vector ever existed for it. Written through the raw primitive directly,
// exactly as `GenerationStore.replayFact` (the recovery fold) applies a
// replayed after-image whose vector leg came back null.
const freshId = randomUUID()
await brain.storage.writeNounRaw(freshId, {
metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 },
vector: null
})
// writeNounRaw bypasses count bookkeeping on purpose (its own JSDoc) — the
// sanctioned recount brings the ledger scalar to ground truth. This walk
// was ALREADY metadata.json-keyed before this fix (rebuildTypeCounts), so
// the recount's answer does not depend on today's change.
await brain.repairIndex()
const ledger = await brain.storage.getCanonicalCounts()
const walk = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
expect(walk.items.length).toBe(ledger.nouns.all)
expect(walk.totalCount).toBe(ledger.nouns.all)
const yielded = walk.items.find((n: any) => n.id === freshId)
expect(yielded, 'the metadata-only row must be YIELDED, not merely counted').toBeDefined()
expect(yielded.vector).toEqual([])
})
it('(a-ids) getNounIdsWithPagination (the zero-read unfiltered enumerator) also yields the metadata-only row', async () => {
await brain.add({ data: 'one', type: 'document' })
await brain.flush()
const freshId = randomUUID()
await brain.storage.writeNounRaw(freshId, {
metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 },
vector: null
})
await brain.repairIndex()
const ledger = await brain.storage.getCanonicalCounts()
const page = await brain.storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
expect(page.ids.length).toBe(ledger.nouns.all)
expect(page.ids).toContain(freshId)
})
it('(b) verbs.all counts a fold-born metadata-only row; the walk yields it when endpoints are recoverable from metadata, and loudly skips (never fabricates) when they are not', async () => {
const a = await brain.add({ data: 'a', type: 'document' })
const b = await brain.add({ data: 'b', type: 'document' })
await brain.relate({ from: a, to: b, type: 'relatedTo' })
await brain.flush()
// Case 1 — the REALISTIC production shape: metadata carries the verb
// type (a reserved field, kept for backward compat) but never
// sourceId/targetId — those are HNSWVerb's structural core and live
// ONLY in the vector leg. The walk cannot safely fabricate them (an
// empty-string endpoint would silently create a phantom edge), so this
// row is counted by the ledger but not yielded — a documented,
// loudly-logged gap, not a silent one.
const gapId = randomUUID()
await brain.storage.writeVerbRaw(gapId, {
metadata: { verb: 'relatedTo', createdAt: Date.now(), updatedAt: Date.now(), weight: 1 },
vector: null
})
// Case 2 — endpoints ARE recoverable from metadata (never true for a
// current production write; modeled here as what a repair tool or a
// future schema could supply): the walk reconstructs and yields it.
const recoveredId = randomUUID()
await brain.storage.writeVerbRaw(recoveredId, {
metadata: {
verb: 'relatedTo',
sourceId: a,
targetId: b,
createdAt: Date.now(),
updatedAt: Date.now(),
weight: 1
},
vector: null
})
await brain.repairIndex()
const ledger = await brain.storage.getCanonicalCounts()
const walk = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } })
// The ledger counts every identity record — the real edge plus both
// synthetic metadata-only rows — unaffected by whether the walk can
// safely hydrate them.
expect(ledger.verbs.all).toBe(3)
const recovered = walk.items.find((v: any) => v.id === recoveredId)
expect(recovered, 'endpoints recoverable from metadata must be yielded').toBeDefined()
expect(recovered.sourceId).toBe(a)
expect(recovered.targetId).toBe(b)
expect(recovered.vector).toEqual([])
// The documented gap: counted, not yielded — this is the one corner of
// the population law this task does NOT close (see the session report).
const gapped = walk.items.find((v: any) => v.id === gapId)
expect(gapped).toBeUndefined()
expect(walk.items.length).toBeLessThan(ledger.verbs.all)
})
})
describe('enumeration population law — LEG 2 (fold preserve-if-absent, below the Brainy API)', () => {
/** A GenerationStore whose brain has already flipped to log authority the
* precondition for `replayFact` (the recovery fold) to run at open(). */
async function openLogAuthorityStore(): Promise<{ storage: MemoryStorage; store: GenerationStore }> {
const storage = new MemoryStorage()
await storage.init()
await storage.writeRawObject(LOG_AUTHORITY_PATH, { authority: 'log' })
const store = new GenerationStore(storage)
await store.open()
return { storage, store }
}
it('(c) nouns: a metadata-only after-image replayed over a vectored row PRESERVES the vector; it stays readable and the vectored ledger is untouched either way', async () => {
const { storage, store } = await openLogAuthorityStore()
const id = randomUUID()
const vectorRecord = { id, vector: [0.1, 0.2, 0.3], connections: {}, level: 0 }
// Generation 1 — a real, honest commit: both legs land together.
await store.commitTransaction({
touched: { nouns: [id], verbs: [] },
execute: async () => {
await storage.writeNounRaw(id, {
metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 },
vector: vectorRecord
})
}
})
const beforeVectoredCount = (await storage.getCanonicalCounts()).vectors.all
// THE ANOMALOUS FACT, crafted directly (bypassing commitTransaction,
// whose honest read-after-write could never produce this on its own):
// metadata changed, vector leg null, while the row is STILL vectored on
// disk. This is exactly the shape the recovery fold must tolerate —
// modeling the confirmed production defect at the replay boundary.
const factLog = store.getFactLog()!
const anomalousFact: CommitFact = {
generation: 2,
timestamp: Date.now(),
ops: [
{
kind: 'noun',
id,
record: {
metadata: { noun: 'document', createdAt: 1000, updatedAt: 2000, _rev: 2 },
vector: null
}
}
]
}
await factLog.append(anomalousFact)
await factLog.sync()
// Reopen — a fresh GenerationStore over the SAME storage. Generation 2's
// fact sits above the (still generation-1) manifest, so it replays
// through the recovery fold — `replayFact`'s own call site.
const store2 = new GenerationStore(storage)
await store2.open()
const after = await storage.readNounRaw(id)
expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull()
expect((after.vector as { vector: number[] }).vector).toEqual([0.1, 0.2, 0.3])
expect((after.metadata as { updatedAt: number }).updatedAt).toBe(2000) // the new metadata DID apply
// writeNounRaw bypasses ledger bookkeeping either way (by design — see
// its JSDoc), so this scalar is unaffected by the replay regardless of
// outcome; asserted for completeness against the task's exact wording.
const afterVectoredCount = (await storage.getCanonicalCounts()).vectors.all
expect(afterVectoredCount).toBe(beforeVectoredCount)
})
it('(c-verb) verbs: a metadata-only after-image replayed over a vectored edge PRESERVES the vector leg (sourceId/targetId/verb intact)', async () => {
const { storage, store } = await openLogAuthorityStore()
const id = randomUUID()
const sourceId = randomUUID()
const targetId = randomUUID()
const vectorRecord = { id, vector: [0.7, 0.8], connections: {}, verb: 'relatedTo', sourceId, targetId }
await store.commitTransaction({
touched: { nouns: [], verbs: [id] },
execute: async () => {
await storage.writeVerbRaw(id, {
metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 1000, weight: 1 },
vector: vectorRecord
})
}
})
const factLog = store.getFactLog()!
const anomalousFact: CommitFact = {
generation: 2,
timestamp: Date.now(),
ops: [
{
kind: 'verb',
id,
record: {
metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 2000, weight: 2 },
vector: null
}
}
]
}
await factLog.append(anomalousFact)
await factLog.sync()
const store2 = new GenerationStore(storage)
await store2.open()
const after = await storage.readVerbRaw(id)
expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull()
expect((after.vector as { sourceId: string }).sourceId).toBe(sourceId)
expect((after.vector as { targetId: string }).targetId).toBe(targetId)
expect((after.metadata as { weight: number }).weight).toBe(2)
})
it('(d) a genuine tombstone replay removes BOTH legs (never preserved)', async () => {
const { storage, store } = await openLogAuthorityStore()
const id = randomUUID()
const vectorRecord = { id, vector: [0.4, 0.5, 0.6], connections: {}, level: 0 }
await store.commitTransaction({
touched: { nouns: [id], verbs: [] },
execute: async () => {
await storage.writeNounRaw(id, {
metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 },
vector: vectorRecord
})
}
})
expect((await storage.readNounRaw(id)).vector).not.toBeNull() // sanity: it landed
const factLog = store.getFactLog()!
await factLog.append({
generation: 2,
timestamp: Date.now(),
ops: [{ kind: 'noun', id, record: null }] // a genuine tombstone — both legs absent
})
await factLog.sync()
const store2 = new GenerationStore(storage)
await store2.open()
const after = await storage.readNounRaw(id)
expect(after.metadata, 'a genuine delete removes the metadata leg').toBeNull()
expect(after.vector, 'a genuine delete removes the vector leg too — preserve-if-absent never applies to a tombstone').toBeNull()
})
})