soulcraftlabs/open-brainy
Archived
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 345 Packages Wiki Activity

Merge branch 'next/enumeration-identity-rekey'

Browse source
This commit is contained in:
David Snelling 2026-08-27 12:39:56 -07:00
commit 204d74c161
3 changed files with 519 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

41
src/db/generationStore.ts
View file

@ -654,10 +654,43 @@ export class GenerationStore {
let replayed = 0
const replayFact = async (fact: CommitFact): Promise<void> => {
for (const op of fact.ops) {
const image =
op.record === null
? { metadata: null, vector: null }
: { metadata: op.record.metadata, vector: op.record.vector }
let image: { metadata: unknown | null; vector: unknown | null }
if (op.record === null) {
// A genuine tombstone (both legs absent) — the fold removes
// both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw
// exact-restore contract.
image = { metadata: null, vector: null }
} else if (
op.record.metadata !== null &&
(op.record.vector === null || op.record.vector === undefined)
) {
// PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's
// half): a metadata-only after-image must never DELETE an
// existing vector leg through the fold. `writeNounRaw`/
// `writeVerbRaw` are exact-restore primitives — a `vector:
// null` there means "delete", which is exactly right for
// `rollBackUncommittedGeneration`'s before-image restore (a
// transaction abort legitimately un-writes a vector the failed
// transaction added). It is NOT right here: this fold replays
// AFTER-IMAGES, and re-applying an already-intact record must
// be byte-safe (this module's own invariant, see the log-authority
// comment above) — silently erasing a landed vector because one
// replayed fact's vector leg came back null is the exact defect
// that left metadata-counted, never-enumerated rows in a
// production store (confirmed root cause: the enumeration walk
// used to key on the vector leg, so a preserved-but-then-deleted
// vector made the row invisible while the ledger still counted
// it by metadata). A genuine "unvector" has its own sanctioned,
// ledger-correct path (`Brainy.unvectorNounForRootMigration`) —
// never this raw primitive, and never the fold.
const current =
op.kind === 'verb'
? await this.storage.readVerbRaw(op.id)
: await this.storage.readNounRaw(op.id)
image = { metadata: op.record.metadata, vector: current.vector ?? null }
} else {
image = { metadata: op.record.metadata, vector: op.record.vector }
}
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
else await this.storage.writeNounRaw(op.id, image)
this.noteCheckpointDirty(op.kind, op.id)

170
src/storage/baseStorage.ts
View file

@ -203,6 +203,40 @@ function idFromVectorPath(path: string): string {
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
}
/**
* @description Extract the entity id embedded in a metadata path
* (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) — the IDENTITY-RECORD
* mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their
* population on this file (ADR-008 G1: the metadata record IS the population;
* the vector leg is optional), so walk ordering and cursor resume derive the
* id from THIS path, never the vector path — a row with metadata and no
* vector file must still be listed, ordered, and resumable.
* @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`).
* @returns The entity id (the path segment immediately before `/metadata.json`).
*/
function idFromMetadataPath(path: string): string {
const withoutSuffix = path.replace(/\/metadata\.json$/, '')
const lastSlash = withoutSuffix.lastIndexOf('/')
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
}
/**
* @description The sanctioned UNVECTORED shape for a noun hydrated during
* enumeration when its identity record (metadata.json) exists but its vector
* leg (vectors.json) does not — a fold-born metadata-only after-image, or any
* row genuinely without a vector yet. Mirrors the shape
* `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned
* unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a
* walk-yielded unvectored row is byte-shape-identical to one produced by that
* migration. Callers already handle `vector: []` as first-class
* (validateAddParams exempts it; index gates key on `length > 0`).
* @param id - The noun id.
* @returns A structurally-valid, vector-empty `HNSWNoun`.
*/
function unvectoredNoun(id: string): HNSWNoun {
return { id, vector: [], connections: new Map<number, Set<string>>(), level: 0 }
}
/**
* Get ID-first path for verb metadata
* No type parameter needed - direct O(1) lookup by ID
@ -1453,6 +1487,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
* rollups are derived state with their own rebuild paths
* (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`).
*
* EXACT-RESTORE PRIMITIVE — `vector: null` DELETES the vector leg, on
* purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on
* this to legitimately un-write a vector a failed transaction added. This
* is deliberately NOT "preserve if absent" — a caller replaying an
* AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must
* apply preserve-if-absent itself BEFORE calling this, by reading the
* current vector and carrying it forward when the after-image's own
* vector leg is null/undefined but its metadata is not (see `replayFact`
* for the implementation and full rationale). A caller that genuinely
* wants to unvector a row uses the sanctioned, ledger-correct path
* (`Brainy.unvectorNounForRootMigration`) — never this primitive.
*
* @param id - The entity id.
* @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}.
*/
@ -1489,7 +1535,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
/**
* Restore a relationship's raw stored objects byte-for-byte (verb-side
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats).
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats,
* same EXACT-RESTORE contract — `vector: null` deletes, on purpose; the
* fold's preserve-if-absent logic lives at its call site, not here).
*
* @param id - The relationship id.
* @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}.
@ -2183,9 +2231,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Stable within-shard order (by noun id) so offset windows and cursor resume
// are deterministic; ids come from the path so skipped nouns are never read.
//
// IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record
// (not the vector) IS the population — a noun with metadata and no vector
// file (a fold-born after-image, see writeNounRaw's preserve-if-absent
// contract) must still enumerate. Keying on metadata.json here means the
// ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed)
// and this walk agree on population by construction. Ordering is
// unaffected for a healthy store: every vectored noun has both legs, so
// the id set and sort order are identical to the old vectors.json keying.
const entries = nounFiles
.filter((p) => p.includes('/vectors.json'))
.map((p) => ({ path: p, id: idFromVectorPath(p) }))
.filter((p) => p.includes('/metadata.json'))
.map((p) => ({ path: p, id: idFromMetadataPath(p) }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
@ -2210,13 +2267,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
) {
const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
const hydrated = await Promise.all(
batch.map(async ({ path: nounPath }) => {
batch.map(async ({ path: metadataPath, id }) => {
try {
const noun = await this.readCanonicalObject(nounPath)
if (!noun) return null
const deserialized = this.deserializeNoun(noun)
const metadata = await this.getNounMetadata(deserialized.id)
const metadata = await this.readCanonicalObject(metadataPath)
if (!metadata) return null
// The vector leg is OPTIONAL (population law): a metadata-only
// row hydrates with the sanctioned unvectored shape rather than
// being dropped from the walk. A fault reading the vector leg
// is treated the same as absence — best-effort, matching the
// canonical recount's tolerance for an unreadable vectors.json
// (rebuildTypeCounts) — a vector-leg problem never hides an
// otherwise-good identity record.
let deserialized: HNSWNoun
try {
const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id))
deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id)
} catch {
deserialized = unvectoredNoun(id)
}
return { deserialized, metadata }
} catch (error) {
// A TORN record must surface typed — a paginated read that
@ -2226,7 +2294,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// walk's job is to HEAL PAST it — skip the victim, serve the rest.
// Identity point-reads (get-by-id) still throw typed upstream.
if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ }
// Skip nouns that fail to load
// Skip nouns whose IDENTITY record fails to load (the metadata
// read above) — that is the one leg this walk cannot proceed
// without.
return null
}
})
@ -2347,9 +2417,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const shardDir = `entities/nouns/${shardHex}`
try {
const nounFiles = await this.listCanonicalObjects(shardDir)
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching
// comment in getNounsWithPagination: metadata.json is the population;
// the vector leg is optional, so a metadata-only row must still be
// listed (and here, for the unfiltered case, needs ZERO reads either
// way — the id comes straight from the path).
const entries = nounFiles
.filter((p) => p.includes('/vectors.json'))
.map((p) => idFromVectorPath(p))
.filter((p) => p.includes('/metadata.json'))
.map((p) => idFromMetadataPath(p))
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
const toWalk =
cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
@ -2560,23 +2635,79 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Stable within-shard order (by verb id) so offset windows and cursor resume
// are deterministic and consistent across calls. Ids come from the path, so
// verbs skipped by the cursor are never read.
//
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of
// this comment in getNounsWithPagination applies here too: metadata.json
// is the population; keying on it here means this walk and the ledger
// recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed)
// agree on population by construction. Unchanged for a healthy store —
// `relate()` always writes both legs of a verb in the same commit, so
// the id set and order match the old vectors.json keying exactly; this
// only additionally surfaces a fold-born metadata-only row (see
// writeVerbRaw's preserve-if-absent contract).
const entries = verbFiles
.filter((p) => p.includes('/vectors.json'))
.map((p) => ({ path: p, id: idFromVectorPath(p) }))
.filter((p) => p.includes('/metadata.json'))
.map((p) => ({ path: p, id: idFromMetadataPath(p) }))
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
for (const { path: verbPath, id: verbId } of entries) {
for (const { path: metadataPath, id: verbId } of entries) {
if (collected.length >= peekCount) break
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
// (later shards are processed in full). No read for skipped verbs.
if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
try {
const rawVerb = await this.readCanonicalObject(verbPath)
if (!rawVerb) continue
// Identity leg first — required. A verb this walk cannot read
// metadata for cannot be hydrated at all (same as before).
const metadata = await this.readCanonicalObject(metadataPath)
if (!metadata) continue
// Deserialize connections Map from JSON storage format
const verb = this.deserializeVerb(rawVerb)
// The vector leg is the verb's STRUCTURAL core (verb/sourceId/
// targetId live there — see coreTypes.ts HNSWVerb), unlike a
// noun's vector, which is pure embedding data. `relate()` always
// writes both legs atomically and verbs have no deferred-embed
// path, so a healthy store's verbs always have both. A vector-leg
// absence here can only be a fold-born after-image (see
// writeVerbRaw's preserve-if-absent contract) — and unlike a
// noun, this walk cannot safely FABRICATE sourceId/targetId to
// synthesize a structurally-valid verb (an empty-string endpoint
// would silently create a phantom edge — worse than omission).
// If the metadata record happens to carry its own sourceId/
// targetId (never true for current production writes, but not
// disallowed — e.g. a future schema or a repair tool could
// populate them), reconstruct from those; otherwise this row is
// loudly skipped — counted by the ledger, but not returned as an
// item, until a repair can supply the missing endpoints.
const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId))
let verb: HNSWVerb
if (rawVerb) {
verb = this.deserializeVerb(rawVerb)
} else {
const metaSourceId = (metadata as Record<string, unknown>).sourceId
const metaTargetId = (metadata as Record<string, unknown>).targetId
const metaVerbType = (metadata as Record<string, unknown>).verb
if (
typeof metaSourceId === 'string' && metaSourceId.length > 0 &&
typeof metaTargetId === 'string' && metaTargetId.length > 0 &&
typeof metaVerbType === 'string' && metaVerbType.length > 0
) {
verb = {
id: verbId,
vector: [],
connections: new Map<number, Set<string>>(),
verb: metaVerbType as VerbType,
sourceId: metaSourceId,
targetId: metaTargetId
}
} else {
prodLog.error(
`[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` +
`record but no vector leg and no recoverable sourceId/targetId — ` +
`skipping (counted by the ledger, not yielded; needs repair).`
)
continue
}
}
// Apply type filter
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
@ -2593,9 +2724,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
continue
}
// Load metadata
const metadata = await this.getVerbMetadata(verb.id)
// Apply subtype filter (requires metadata — checked AFTER load)
if (filterSubtypes) {
const subtype = metadata?.subtype as string | undefined