Merge branch 'next/enumeration-identity-rekey'
This commit is contained in:
commit
204d74c161
3 changed files with 519 additions and 25 deletions
|
|
@ -654,10 +654,43 @@ export class GenerationStore {
|
||||||
let replayed = 0
|
let replayed = 0
|
||||||
const replayFact = async (fact: CommitFact): Promise<void> => {
|
const replayFact = async (fact: CommitFact): Promise<void> => {
|
||||||
for (const op of fact.ops) {
|
for (const op of fact.ops) {
|
||||||
const image =
|
let image: { metadata: unknown | null; vector: unknown | null }
|
||||||
op.record === null
|
if (op.record === null) {
|
||||||
? { metadata: null, vector: null }
|
// A genuine tombstone (both legs absent) — the fold removes
|
||||||
: { metadata: op.record.metadata, vector: op.record.vector }
|
// both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw
|
||||||
|
// exact-restore contract.
|
||||||
|
image = { metadata: null, vector: null }
|
||||||
|
} else if (
|
||||||
|
op.record.metadata !== null &&
|
||||||
|
(op.record.vector === null || op.record.vector === undefined)
|
||||||
|
) {
|
||||||
|
// PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's
|
||||||
|
// half): a metadata-only after-image must never DELETE an
|
||||||
|
// existing vector leg through the fold. `writeNounRaw`/
|
||||||
|
// `writeVerbRaw` are exact-restore primitives — a `vector:
|
||||||
|
// null` there means "delete", which is exactly right for
|
||||||
|
// `rollBackUncommittedGeneration`'s before-image restore (a
|
||||||
|
// transaction abort legitimately un-writes a vector the failed
|
||||||
|
// transaction added). It is NOT right here: this fold replays
|
||||||
|
// AFTER-IMAGES, and re-applying an already-intact record must
|
||||||
|
// be byte-safe (this module's own invariant, see the log-authority
|
||||||
|
// comment above) — silently erasing a landed vector because one
|
||||||
|
// replayed fact's vector leg came back null is the exact defect
|
||||||
|
// that left metadata-counted, never-enumerated rows in a
|
||||||
|
// production store (confirmed root cause: the enumeration walk
|
||||||
|
// used to key on the vector leg, so a preserved-but-then-deleted
|
||||||
|
// vector made the row invisible while the ledger still counted
|
||||||
|
// it by metadata). A genuine "unvector" has its own sanctioned,
|
||||||
|
// ledger-correct path (`Brainy.unvectorNounForRootMigration`) —
|
||||||
|
// never this raw primitive, and never the fold.
|
||||||
|
const current =
|
||||||
|
op.kind === 'verb'
|
||||||
|
? await this.storage.readVerbRaw(op.id)
|
||||||
|
: await this.storage.readNounRaw(op.id)
|
||||||
|
image = { metadata: op.record.metadata, vector: current.vector ?? null }
|
||||||
|
} else {
|
||||||
|
image = { metadata: op.record.metadata, vector: op.record.vector }
|
||||||
|
}
|
||||||
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
|
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
|
||||||
else await this.storage.writeNounRaw(op.id, image)
|
else await this.storage.writeNounRaw(op.id, image)
|
||||||
this.noteCheckpointDirty(op.kind, op.id)
|
this.noteCheckpointDirty(op.kind, op.id)
|
||||||
|
|
|
||||||
|
|
@ -203,6 +203,40 @@ function idFromVectorPath(path: string): string {
|
||||||
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
|
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description Extract the entity id embedded in a metadata path
|
||||||
|
* (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) — the IDENTITY-RECORD
|
||||||
|
* mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their
|
||||||
|
* population on this file (ADR-008 G1: the metadata record IS the population;
|
||||||
|
* the vector leg is optional), so walk ordering and cursor resume derive the
|
||||||
|
* id from THIS path, never the vector path — a row with metadata and no
|
||||||
|
* vector file must still be listed, ordered, and resumable.
|
||||||
|
* @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`).
|
||||||
|
* @returns The entity id (the path segment immediately before `/metadata.json`).
|
||||||
|
*/
|
||||||
|
function idFromMetadataPath(path: string): string {
|
||||||
|
const withoutSuffix = path.replace(/\/metadata\.json$/, '')
|
||||||
|
const lastSlash = withoutSuffix.lastIndexOf('/')
|
||||||
|
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The sanctioned UNVECTORED shape for a noun hydrated during
|
||||||
|
* enumeration when its identity record (metadata.json) exists but its vector
|
||||||
|
* leg (vectors.json) does not — a fold-born metadata-only after-image, or any
|
||||||
|
* row genuinely without a vector yet. Mirrors the shape
|
||||||
|
* `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned
|
||||||
|
* unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a
|
||||||
|
* walk-yielded unvectored row is byte-shape-identical to one produced by that
|
||||||
|
* migration. Callers already handle `vector: []` as first-class
|
||||||
|
* (validateAddParams exempts it; index gates key on `length > 0`).
|
||||||
|
* @param id - The noun id.
|
||||||
|
* @returns A structurally-valid, vector-empty `HNSWNoun`.
|
||||||
|
*/
|
||||||
|
function unvectoredNoun(id: string): HNSWNoun {
|
||||||
|
return { id, vector: [], connections: new Map<number, Set<string>>(), level: 0 }
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get ID-first path for verb metadata
|
* Get ID-first path for verb metadata
|
||||||
* No type parameter needed - direct O(1) lookup by ID
|
* No type parameter needed - direct O(1) lookup by ID
|
||||||
|
|
@ -1453,6 +1487,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
* rollups are derived state with their own rebuild paths
|
* rollups are derived state with their own rebuild paths
|
||||||
* (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`).
|
* (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`).
|
||||||
*
|
*
|
||||||
|
* EXACT-RESTORE PRIMITIVE — `vector: null` DELETES the vector leg, on
|
||||||
|
* purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on
|
||||||
|
* this to legitimately un-write a vector a failed transaction added. This
|
||||||
|
* is deliberately NOT "preserve if absent" — a caller replaying an
|
||||||
|
* AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must
|
||||||
|
* apply preserve-if-absent itself BEFORE calling this, by reading the
|
||||||
|
* current vector and carrying it forward when the after-image's own
|
||||||
|
* vector leg is null/undefined but its metadata is not (see `replayFact`
|
||||||
|
* for the implementation and full rationale). A caller that genuinely
|
||||||
|
* wants to unvector a row uses the sanctioned, ledger-correct path
|
||||||
|
* (`Brainy.unvectorNounForRootMigration`) — never this primitive.
|
||||||
|
*
|
||||||
* @param id - The entity id.
|
* @param id - The entity id.
|
||||||
* @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}.
|
* @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}.
|
||||||
*/
|
*/
|
||||||
|
|
@ -1489,7 +1535,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Restore a relationship's raw stored objects byte-for-byte (verb-side
|
* Restore a relationship's raw stored objects byte-for-byte (verb-side
|
||||||
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats).
|
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats,
|
||||||
|
* same EXACT-RESTORE contract — `vector: null` deletes, on purpose; the
|
||||||
|
* fold's preserve-if-absent logic lives at its call site, not here).
|
||||||
*
|
*
|
||||||
* @param id - The relationship id.
|
* @param id - The relationship id.
|
||||||
* @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}.
|
* @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}.
|
||||||
|
|
@ -2183,9 +2231,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
// Stable within-shard order (by noun id) so offset windows and cursor resume
|
// Stable within-shard order (by noun id) so offset windows and cursor resume
|
||||||
// are deterministic; ids come from the path so skipped nouns are never read.
|
// are deterministic; ids come from the path so skipped nouns are never read.
|
||||||
|
//
|
||||||
|
// IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record
|
||||||
|
// (not the vector) IS the population — a noun with metadata and no vector
|
||||||
|
// file (a fold-born after-image, see writeNounRaw's preserve-if-absent
|
||||||
|
// contract) must still enumerate. Keying on metadata.json here means the
|
||||||
|
// ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed)
|
||||||
|
// and this walk agree on population by construction. Ordering is
|
||||||
|
// unaffected for a healthy store: every vectored noun has both legs, so
|
||||||
|
// the id set and sort order are identical to the old vectors.json keying.
|
||||||
const entries = nounFiles
|
const entries = nounFiles
|
||||||
.filter((p) => p.includes('/vectors.json'))
|
.filter((p) => p.includes('/metadata.json'))
|
||||||
.map((p) => ({ path: p, id: idFromVectorPath(p) }))
|
.map((p) => ({ path: p, id: idFromMetadataPath(p) }))
|
||||||
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
||||||
|
|
||||||
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
|
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
|
||||||
|
|
@ -2210,13 +2267,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
) {
|
) {
|
||||||
const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
|
const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
|
||||||
const hydrated = await Promise.all(
|
const hydrated = await Promise.all(
|
||||||
batch.map(async ({ path: nounPath }) => {
|
batch.map(async ({ path: metadataPath, id }) => {
|
||||||
try {
|
try {
|
||||||
const noun = await this.readCanonicalObject(nounPath)
|
const metadata = await this.readCanonicalObject(metadataPath)
|
||||||
if (!noun) return null
|
|
||||||
const deserialized = this.deserializeNoun(noun)
|
|
||||||
const metadata = await this.getNounMetadata(deserialized.id)
|
|
||||||
if (!metadata) return null
|
if (!metadata) return null
|
||||||
|
// The vector leg is OPTIONAL (population law): a metadata-only
|
||||||
|
// row hydrates with the sanctioned unvectored shape rather than
|
||||||
|
// being dropped from the walk. A fault reading the vector leg
|
||||||
|
// is treated the same as absence — best-effort, matching the
|
||||||
|
// canonical recount's tolerance for an unreadable vectors.json
|
||||||
|
// (rebuildTypeCounts) — a vector-leg problem never hides an
|
||||||
|
// otherwise-good identity record.
|
||||||
|
let deserialized: HNSWNoun
|
||||||
|
try {
|
||||||
|
const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id))
|
||||||
|
deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id)
|
||||||
|
} catch {
|
||||||
|
deserialized = unvectoredNoun(id)
|
||||||
|
}
|
||||||
return { deserialized, metadata }
|
return { deserialized, metadata }
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// A TORN record must surface typed — a paginated read that
|
// A TORN record must surface typed — a paginated read that
|
||||||
|
|
@ -2226,7 +2294,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// walk's job is to HEAL PAST it — skip the victim, serve the rest.
|
// walk's job is to HEAL PAST it — skip the victim, serve the rest.
|
||||||
// Identity point-reads (get-by-id) still throw typed upstream.
|
// Identity point-reads (get-by-id) still throw typed upstream.
|
||||||
if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ }
|
if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ }
|
||||||
// Skip nouns that fail to load
|
// Skip nouns whose IDENTITY record fails to load (the metadata
|
||||||
|
// read above) — that is the one leg this walk cannot proceed
|
||||||
|
// without.
|
||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
@ -2347,9 +2417,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const shardDir = `entities/nouns/${shardHex}`
|
const shardDir = `entities/nouns/${shardHex}`
|
||||||
try {
|
try {
|
||||||
const nounFiles = await this.listCanonicalObjects(shardDir)
|
const nounFiles = await this.listCanonicalObjects(shardDir)
|
||||||
|
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching
|
||||||
|
// comment in getNounsWithPagination: metadata.json is the population;
|
||||||
|
// the vector leg is optional, so a metadata-only row must still be
|
||||||
|
// listed (and here, for the unfiltered case, needs ZERO reads either
|
||||||
|
// way — the id comes straight from the path).
|
||||||
const entries = nounFiles
|
const entries = nounFiles
|
||||||
.filter((p) => p.includes('/vectors.json'))
|
.filter((p) => p.includes('/metadata.json'))
|
||||||
.map((p) => idFromVectorPath(p))
|
.map((p) => idFromMetadataPath(p))
|
||||||
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
|
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
|
||||||
const toWalk =
|
const toWalk =
|
||||||
cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
|
cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
|
||||||
|
|
@ -2560,23 +2635,79 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// Stable within-shard order (by verb id) so offset windows and cursor resume
|
// Stable within-shard order (by verb id) so offset windows and cursor resume
|
||||||
// are deterministic and consistent across calls. Ids come from the path, so
|
// are deterministic and consistent across calls. Ids come from the path, so
|
||||||
// verbs skipped by the cursor are never read.
|
// verbs skipped by the cursor are never read.
|
||||||
|
//
|
||||||
|
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of
|
||||||
|
// this comment in getNounsWithPagination applies here too: metadata.json
|
||||||
|
// is the population; keying on it here means this walk and the ledger
|
||||||
|
// recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed)
|
||||||
|
// agree on population by construction. Unchanged for a healthy store —
|
||||||
|
// `relate()` always writes both legs of a verb in the same commit, so
|
||||||
|
// the id set and order match the old vectors.json keying exactly; this
|
||||||
|
// only additionally surfaces a fold-born metadata-only row (see
|
||||||
|
// writeVerbRaw's preserve-if-absent contract).
|
||||||
const entries = verbFiles
|
const entries = verbFiles
|
||||||
.filter((p) => p.includes('/vectors.json'))
|
.filter((p) => p.includes('/metadata.json'))
|
||||||
.map((p) => ({ path: p, id: idFromVectorPath(p) }))
|
.map((p) => ({ path: p, id: idFromMetadataPath(p) }))
|
||||||
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
||||||
|
|
||||||
for (const { path: verbPath, id: verbId } of entries) {
|
for (const { path: metadataPath, id: verbId } of entries) {
|
||||||
if (collected.length >= peekCount) break
|
if (collected.length >= peekCount) break
|
||||||
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
|
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
|
||||||
// (later shards are processed in full). No read for skipped verbs.
|
// (later shards are processed in full). No read for skipped verbs.
|
||||||
if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
|
if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const rawVerb = await this.readCanonicalObject(verbPath)
|
// Identity leg first — required. A verb this walk cannot read
|
||||||
if (!rawVerb) continue
|
// metadata for cannot be hydrated at all (same as before).
|
||||||
|
const metadata = await this.readCanonicalObject(metadataPath)
|
||||||
|
if (!metadata) continue
|
||||||
|
|
||||||
// Deserialize connections Map from JSON storage format
|
// The vector leg is the verb's STRUCTURAL core (verb/sourceId/
|
||||||
const verb = this.deserializeVerb(rawVerb)
|
// targetId live there — see coreTypes.ts HNSWVerb), unlike a
|
||||||
|
// noun's vector, which is pure embedding data. `relate()` always
|
||||||
|
// writes both legs atomically and verbs have no deferred-embed
|
||||||
|
// path, so a healthy store's verbs always have both. A vector-leg
|
||||||
|
// absence here can only be a fold-born after-image (see
|
||||||
|
// writeVerbRaw's preserve-if-absent contract) — and unlike a
|
||||||
|
// noun, this walk cannot safely FABRICATE sourceId/targetId to
|
||||||
|
// synthesize a structurally-valid verb (an empty-string endpoint
|
||||||
|
// would silently create a phantom edge — worse than omission).
|
||||||
|
// If the metadata record happens to carry its own sourceId/
|
||||||
|
// targetId (never true for current production writes, but not
|
||||||
|
// disallowed — e.g. a future schema or a repair tool could
|
||||||
|
// populate them), reconstruct from those; otherwise this row is
|
||||||
|
// loudly skipped — counted by the ledger, but not returned as an
|
||||||
|
// item, until a repair can supply the missing endpoints.
|
||||||
|
const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId))
|
||||||
|
let verb: HNSWVerb
|
||||||
|
if (rawVerb) {
|
||||||
|
verb = this.deserializeVerb(rawVerb)
|
||||||
|
} else {
|
||||||
|
const metaSourceId = (metadata as Record<string, unknown>).sourceId
|
||||||
|
const metaTargetId = (metadata as Record<string, unknown>).targetId
|
||||||
|
const metaVerbType = (metadata as Record<string, unknown>).verb
|
||||||
|
if (
|
||||||
|
typeof metaSourceId === 'string' && metaSourceId.length > 0 &&
|
||||||
|
typeof metaTargetId === 'string' && metaTargetId.length > 0 &&
|
||||||
|
typeof metaVerbType === 'string' && metaVerbType.length > 0
|
||||||
|
) {
|
||||||
|
verb = {
|
||||||
|
id: verbId,
|
||||||
|
vector: [],
|
||||||
|
connections: new Map<number, Set<string>>(),
|
||||||
|
verb: metaVerbType as VerbType,
|
||||||
|
sourceId: metaSourceId,
|
||||||
|
targetId: metaTargetId
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
prodLog.error(
|
||||||
|
`[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` +
|
||||||
|
`record but no vector leg and no recoverable sourceId/targetId — ` +
|
||||||
|
`skipping (counted by the ledger, not yielded; needs repair).`
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Apply type filter
|
// Apply type filter
|
||||||
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
|
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
|
||||||
|
|
@ -2593,9 +2724,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load metadata
|
|
||||||
const metadata = await this.getVerbMetadata(verb.id)
|
|
||||||
|
|
||||||
// Apply subtype filter (requires metadata — checked AFTER load)
|
// Apply subtype filter (requires metadata — checked AFTER load)
|
||||||
if (filterSubtypes) {
|
if (filterSubtypes) {
|
||||||
const subtype = metadata?.subtype as string | undefined
|
const subtype = metadata?.subtype as string | undefined
|
||||||
|
|
|
||||||
333
tests/integration/enumeration-population-law.test.ts
Normal file
333
tests/integration/enumeration-population-law.test.ts
Normal file
|
|
@ -0,0 +1,333 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/enumeration-population-law
|
||||||
|
* @description THE POPULATION LAW (ADR-008 G1): the unfiltered noun/verb walk
|
||||||
|
* and the canonical ALL scalar must agree on the population — a row's
|
||||||
|
* IDENTITY RECORD (metadata.json) is what defines membership; the vector leg
|
||||||
|
* is optional data, never a gate on visibility. Before this fix, the walk
|
||||||
|
* (getNounsWithPagination / getNounIdsWithPagination / getVerbsWithPagination)
|
||||||
|
* enumerated by keying on the VECTOR leg (`vectors.json`), so a row with
|
||||||
|
* metadata and no vector file was counted by the ledger (already
|
||||||
|
* metadata.json-keyed — see `rebuildTypeCounts`) but never yielded by the
|
||||||
|
* walk: a permanent "counted but invisible" phantom for any downstream
|
||||||
|
* consumer (a health-coverage row, an index-fill walk) that iterates the walk
|
||||||
|
* to account for the ledger's total.
|
||||||
|
*
|
||||||
|
* Two legs are pinned here:
|
||||||
|
* (a)/(b) LEG 1 — the walk re-keys on metadata.json. A fold-born
|
||||||
|
* metadata-only row (the exact shape `GenerationStore.replayFact` can
|
||||||
|
* leave behind, and the exact shape `writeNounRaw`/`writeVerbRaw` accept)
|
||||||
|
* must be YIELDED, hydrated with the sanctioned unvectored shape
|
||||||
|
* (`vector: []`) — not merely counted.
|
||||||
|
*
|
||||||
|
* For VERBS this closes only PARTIALLY: `sourceId`/`targetId` are
|
||||||
|
* HNSWVerb's structural core and live ONLY in the vector leg (never in
|
||||||
|
* metadata — see `RESERVED_RELATION_FIELDS` in reservedFields.ts, which
|
||||||
|
* does not include them). A metadata-only verb row therefore cannot be
|
||||||
|
* safely reconstructed without FABRICATING an edge's endpoints — which
|
||||||
|
* would silently create a phantom relationship, strictly worse than the
|
||||||
|
* original defect. The walk recovers the row when its metadata happens
|
||||||
|
* to carry `sourceId`/`targetId` (a defensive, forward-compatible
|
||||||
|
* fallback — never true for a CURRENT production write, but not
|
||||||
|
* disallowed either); otherwise it counts the row (ledger, unchanged)
|
||||||
|
* but loudly skips yielding it, logging the gap instead of hiding it.
|
||||||
|
* Closing this fully requires persisting `sourceId`/`targetId` in verb
|
||||||
|
* metadata — a schema change out of this task's scope; see the session
|
||||||
|
* report for the explicit call-out.
|
||||||
|
*
|
||||||
|
* (c)/(d) LEG 2 — the recovery fold's preserve-if-absent contract, exercised
|
||||||
|
* directly against `GenerationStore`/`FactLog` (below the `Brainy` API):
|
||||||
|
* a metadata-only after-image replayed over an already-vectored row must
|
||||||
|
* PRESERVE the existing vector leg (never delete it); a genuine tombstone
|
||||||
|
* (both legs absent) still removes both legs.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { randomUUID } from 'node:crypto'
|
||||||
|
import { Brainy } from '../../src/index.js'
|
||||||
|
import { GenerationStore } from '../../src/db/generationStore.js'
|
||||||
|
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
|
||||||
|
import { LOG_AUTHORITY_PATH } from '../../src/db/logAuthority.js'
|
||||||
|
import type { CommitFact } from '../../src/db/factLog.js'
|
||||||
|
|
||||||
|
describe('enumeration population law — LEG 1 (identity-keyed walk)', () => {
|
||||||
|
let dir: string
|
||||||
|
let brain: any
|
||||||
|
|
||||||
|
const open = async () => {
|
||||||
|
const b: any = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
silent: true,
|
||||||
|
dimensions: 384
|
||||||
|
})
|
||||||
|
await b.init()
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||||
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-population-law-'))
|
||||||
|
brain = await open()
|
||||||
|
})
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close?.().catch(() => {})
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(a) nouns.all equals the unfiltered walk-yield count with a fold-born metadata-only row present', async () => {
|
||||||
|
// Ordinary, fully-vectored background population.
|
||||||
|
await brain.add({ data: 'one', type: 'document' })
|
||||||
|
await brain.add({ data: 'two', type: 'document' })
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
// THE EXACT PRE-FIX SHAPE: a metadata-only row against a FRESH id — no
|
||||||
|
// vector ever existed for it. Written through the raw primitive directly,
|
||||||
|
// exactly as `GenerationStore.replayFact` (the recovery fold) applies a
|
||||||
|
// replayed after-image whose vector leg came back null.
|
||||||
|
const freshId = randomUUID()
|
||||||
|
await brain.storage.writeNounRaw(freshId, {
|
||||||
|
metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 },
|
||||||
|
vector: null
|
||||||
|
})
|
||||||
|
|
||||||
|
// writeNounRaw bypasses count bookkeeping on purpose (its own JSDoc) — the
|
||||||
|
// sanctioned recount brings the ledger scalar to ground truth. This walk
|
||||||
|
// was ALREADY metadata.json-keyed before this fix (rebuildTypeCounts), so
|
||||||
|
// the recount's answer does not depend on today's change.
|
||||||
|
await brain.repairIndex()
|
||||||
|
|
||||||
|
const ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
const walk = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
|
||||||
|
|
||||||
|
expect(walk.items.length).toBe(ledger.nouns.all)
|
||||||
|
expect(walk.totalCount).toBe(ledger.nouns.all)
|
||||||
|
|
||||||
|
const yielded = walk.items.find((n: any) => n.id === freshId)
|
||||||
|
expect(yielded, 'the metadata-only row must be YIELDED, not merely counted').toBeDefined()
|
||||||
|
expect(yielded.vector).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(a-ids) getNounIdsWithPagination (the zero-read unfiltered enumerator) also yields the metadata-only row', async () => {
|
||||||
|
await brain.add({ data: 'one', type: 'document' })
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
const freshId = randomUUID()
|
||||||
|
await brain.storage.writeNounRaw(freshId, {
|
||||||
|
metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 },
|
||||||
|
vector: null
|
||||||
|
})
|
||||||
|
await brain.repairIndex()
|
||||||
|
|
||||||
|
const ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
const page = await brain.storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
|
||||||
|
expect(page.ids.length).toBe(ledger.nouns.all)
|
||||||
|
expect(page.ids).toContain(freshId)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(b) verbs.all counts a fold-born metadata-only row; the walk yields it when endpoints are recoverable from metadata, and loudly skips (never fabricates) when they are not', async () => {
|
||||||
|
const a = await brain.add({ data: 'a', type: 'document' })
|
||||||
|
const b = await brain.add({ data: 'b', type: 'document' })
|
||||||
|
await brain.relate({ from: a, to: b, type: 'relatedTo' })
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
// Case 1 — the REALISTIC production shape: metadata carries the verb
|
||||||
|
// type (a reserved field, kept for backward compat) but never
|
||||||
|
// sourceId/targetId — those are HNSWVerb's structural core and live
|
||||||
|
// ONLY in the vector leg. The walk cannot safely fabricate them (an
|
||||||
|
// empty-string endpoint would silently create a phantom edge), so this
|
||||||
|
// row is counted by the ledger but not yielded — a documented,
|
||||||
|
// loudly-logged gap, not a silent one.
|
||||||
|
const gapId = randomUUID()
|
||||||
|
await brain.storage.writeVerbRaw(gapId, {
|
||||||
|
metadata: { verb: 'relatedTo', createdAt: Date.now(), updatedAt: Date.now(), weight: 1 },
|
||||||
|
vector: null
|
||||||
|
})
|
||||||
|
|
||||||
|
// Case 2 — endpoints ARE recoverable from metadata (never true for a
|
||||||
|
// current production write; modeled here as what a repair tool or a
|
||||||
|
// future schema could supply): the walk reconstructs and yields it.
|
||||||
|
const recoveredId = randomUUID()
|
||||||
|
await brain.storage.writeVerbRaw(recoveredId, {
|
||||||
|
metadata: {
|
||||||
|
verb: 'relatedTo',
|
||||||
|
sourceId: a,
|
||||||
|
targetId: b,
|
||||||
|
createdAt: Date.now(),
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
weight: 1
|
||||||
|
},
|
||||||
|
vector: null
|
||||||
|
})
|
||||||
|
|
||||||
|
await brain.repairIndex()
|
||||||
|
const ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
const walk = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } })
|
||||||
|
|
||||||
|
// The ledger counts every identity record — the real edge plus both
|
||||||
|
// synthetic metadata-only rows — unaffected by whether the walk can
|
||||||
|
// safely hydrate them.
|
||||||
|
expect(ledger.verbs.all).toBe(3)
|
||||||
|
|
||||||
|
const recovered = walk.items.find((v: any) => v.id === recoveredId)
|
||||||
|
expect(recovered, 'endpoints recoverable from metadata must be yielded').toBeDefined()
|
||||||
|
expect(recovered.sourceId).toBe(a)
|
||||||
|
expect(recovered.targetId).toBe(b)
|
||||||
|
expect(recovered.vector).toEqual([])
|
||||||
|
|
||||||
|
// The documented gap: counted, not yielded — this is the one corner of
|
||||||
|
// the population law this task does NOT close (see the session report).
|
||||||
|
const gapped = walk.items.find((v: any) => v.id === gapId)
|
||||||
|
expect(gapped).toBeUndefined()
|
||||||
|
expect(walk.items.length).toBeLessThan(ledger.verbs.all)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('enumeration population law — LEG 2 (fold preserve-if-absent, below the Brainy API)', () => {
|
||||||
|
/** A GenerationStore whose brain has already flipped to log authority — the
|
||||||
|
* precondition for `replayFact` (the recovery fold) to run at open(). */
|
||||||
|
async function openLogAuthorityStore(): Promise<{ storage: MemoryStorage; store: GenerationStore }> {
|
||||||
|
const storage = new MemoryStorage()
|
||||||
|
await storage.init()
|
||||||
|
await storage.writeRawObject(LOG_AUTHORITY_PATH, { authority: 'log' })
|
||||||
|
const store = new GenerationStore(storage)
|
||||||
|
await store.open()
|
||||||
|
return { storage, store }
|
||||||
|
}
|
||||||
|
|
||||||
|
it('(c) nouns: a metadata-only after-image replayed over a vectored row PRESERVES the vector; it stays readable and the vectored ledger is untouched either way', async () => {
|
||||||
|
const { storage, store } = await openLogAuthorityStore()
|
||||||
|
const id = randomUUID()
|
||||||
|
const vectorRecord = { id, vector: [0.1, 0.2, 0.3], connections: {}, level: 0 }
|
||||||
|
|
||||||
|
// Generation 1 — a real, honest commit: both legs land together.
|
||||||
|
await store.commitTransaction({
|
||||||
|
touched: { nouns: [id], verbs: [] },
|
||||||
|
execute: async () => {
|
||||||
|
await storage.writeNounRaw(id, {
|
||||||
|
metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 },
|
||||||
|
vector: vectorRecord
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
const beforeVectoredCount = (await storage.getCanonicalCounts()).vectors.all
|
||||||
|
|
||||||
|
// THE ANOMALOUS FACT, crafted directly (bypassing commitTransaction,
|
||||||
|
// whose honest read-after-write could never produce this on its own):
|
||||||
|
// metadata changed, vector leg null, while the row is STILL vectored on
|
||||||
|
// disk. This is exactly the shape the recovery fold must tolerate —
|
||||||
|
// modeling the confirmed production defect at the replay boundary.
|
||||||
|
const factLog = store.getFactLog()!
|
||||||
|
const anomalousFact: CommitFact = {
|
||||||
|
generation: 2,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
ops: [
|
||||||
|
{
|
||||||
|
kind: 'noun',
|
||||||
|
id,
|
||||||
|
record: {
|
||||||
|
metadata: { noun: 'document', createdAt: 1000, updatedAt: 2000, _rev: 2 },
|
||||||
|
vector: null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
await factLog.append(anomalousFact)
|
||||||
|
await factLog.sync()
|
||||||
|
|
||||||
|
// Reopen — a fresh GenerationStore over the SAME storage. Generation 2's
|
||||||
|
// fact sits above the (still generation-1) manifest, so it replays
|
||||||
|
// through the recovery fold — `replayFact`'s own call site.
|
||||||
|
const store2 = new GenerationStore(storage)
|
||||||
|
await store2.open()
|
||||||
|
|
||||||
|
const after = await storage.readNounRaw(id)
|
||||||
|
expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull()
|
||||||
|
expect((after.vector as { vector: number[] }).vector).toEqual([0.1, 0.2, 0.3])
|
||||||
|
expect((after.metadata as { updatedAt: number }).updatedAt).toBe(2000) // the new metadata DID apply
|
||||||
|
|
||||||
|
// writeNounRaw bypasses ledger bookkeeping either way (by design — see
|
||||||
|
// its JSDoc), so this scalar is unaffected by the replay regardless of
|
||||||
|
// outcome; asserted for completeness against the task's exact wording.
|
||||||
|
const afterVectoredCount = (await storage.getCanonicalCounts()).vectors.all
|
||||||
|
expect(afterVectoredCount).toBe(beforeVectoredCount)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(c-verb) verbs: a metadata-only after-image replayed over a vectored edge PRESERVES the vector leg (sourceId/targetId/verb intact)', async () => {
|
||||||
|
const { storage, store } = await openLogAuthorityStore()
|
||||||
|
const id = randomUUID()
|
||||||
|
const sourceId = randomUUID()
|
||||||
|
const targetId = randomUUID()
|
||||||
|
const vectorRecord = { id, vector: [0.7, 0.8], connections: {}, verb: 'relatedTo', sourceId, targetId }
|
||||||
|
|
||||||
|
await store.commitTransaction({
|
||||||
|
touched: { nouns: [], verbs: [id] },
|
||||||
|
execute: async () => {
|
||||||
|
await storage.writeVerbRaw(id, {
|
||||||
|
metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 1000, weight: 1 },
|
||||||
|
vector: vectorRecord
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const factLog = store.getFactLog()!
|
||||||
|
const anomalousFact: CommitFact = {
|
||||||
|
generation: 2,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
ops: [
|
||||||
|
{
|
||||||
|
kind: 'verb',
|
||||||
|
id,
|
||||||
|
record: {
|
||||||
|
metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 2000, weight: 2 },
|
||||||
|
vector: null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
await factLog.append(anomalousFact)
|
||||||
|
await factLog.sync()
|
||||||
|
|
||||||
|
const store2 = new GenerationStore(storage)
|
||||||
|
await store2.open()
|
||||||
|
|
||||||
|
const after = await storage.readVerbRaw(id)
|
||||||
|
expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull()
|
||||||
|
expect((after.vector as { sourceId: string }).sourceId).toBe(sourceId)
|
||||||
|
expect((after.vector as { targetId: string }).targetId).toBe(targetId)
|
||||||
|
expect((after.metadata as { weight: number }).weight).toBe(2)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('(d) a genuine tombstone replay removes BOTH legs (never preserved)', async () => {
|
||||||
|
const { storage, store } = await openLogAuthorityStore()
|
||||||
|
const id = randomUUID()
|
||||||
|
const vectorRecord = { id, vector: [0.4, 0.5, 0.6], connections: {}, level: 0 }
|
||||||
|
|
||||||
|
await store.commitTransaction({
|
||||||
|
touched: { nouns: [id], verbs: [] },
|
||||||
|
execute: async () => {
|
||||||
|
await storage.writeNounRaw(id, {
|
||||||
|
metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 },
|
||||||
|
vector: vectorRecord
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
expect((await storage.readNounRaw(id)).vector).not.toBeNull() // sanity: it landed
|
||||||
|
|
||||||
|
const factLog = store.getFactLog()!
|
||||||
|
await factLog.append({
|
||||||
|
generation: 2,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
ops: [{ kind: 'noun', id, record: null }] // a genuine tombstone — both legs absent
|
||||||
|
})
|
||||||
|
await factLog.sync()
|
||||||
|
|
||||||
|
const store2 = new GenerationStore(storage)
|
||||||
|
await store2.open()
|
||||||
|
|
||||||
|
const after = await storage.readNounRaw(id)
|
||||||
|
expect(after.metadata, 'a genuine delete removes the metadata leg').toBeNull()
|
||||||
|
expect(after.vector, 'a genuine delete removes the vector leg too — preserve-if-absent never applies to a tombstone').toBeNull()
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue