fix(storage): enumeration re-keys on the identity record, not the vector leg
The noun/verb pagination walks (getNounsWithPagination, getNounIdsWithPagination, getVerbsWithPagination) listed shard contents by filtering for vectors.json, while the canonical count ledger has always counted a row by its metadata.json presence alone. A row with metadata and no vector file was therefore counted by the ledger but never yielded by the walk — a permanent "counted but invisible" phantom for any downstream consumer that iterates the walk to account for the ledger's total. Nouns now enumerate by metadata.json and hydrate the vector leg optionally, yielding the sanctioned unvectored shape (vector: []) when it's absent. Verbs enumerate the same way, but a metadata-only verb row can only be fully reconstructed when sourceId/targetId happen to be recoverable from metadata (never true for a current production write — those fields live only in the vector leg); otherwise the row is counted but loudly skipped rather than fabricated, since a phantom edge with fake endpoints would be worse than the original defect. Separately, GenerationStore's recovery-fold replay (replayFact) now applies preserve-if-absent: a metadata-only after-image replayed over an already- vectored row carries the existing vector forward instead of deleting it via writeNounRaw/writeVerbRaw's exact-restore null-means-delete contract (which must stay exact for transaction-abort rollback). A genuine tombstone still removes both legs.
This commit is contained in:
parent
4c7b0fab7a
commit
f8d8ce16b9
3 changed files with 519 additions and 25 deletions
|
|
@ -654,10 +654,43 @@ export class GenerationStore {
|
|||
let replayed = 0
|
||||
const replayFact = async (fact: CommitFact): Promise<void> => {
|
||||
for (const op of fact.ops) {
|
||||
const image =
|
||||
op.record === null
|
||||
? { metadata: null, vector: null }
|
||||
: { metadata: op.record.metadata, vector: op.record.vector }
|
||||
let image: { metadata: unknown | null; vector: unknown | null }
|
||||
if (op.record === null) {
|
||||
// A genuine tombstone (both legs absent) — the fold removes
|
||||
// both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw
|
||||
// exact-restore contract.
|
||||
image = { metadata: null, vector: null }
|
||||
} else if (
|
||||
op.record.metadata !== null &&
|
||||
(op.record.vector === null || op.record.vector === undefined)
|
||||
) {
|
||||
// PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's
|
||||
// half): a metadata-only after-image must never DELETE an
|
||||
// existing vector leg through the fold. `writeNounRaw`/
|
||||
// `writeVerbRaw` are exact-restore primitives — a `vector:
|
||||
// null` there means "delete", which is exactly right for
|
||||
// `rollBackUncommittedGeneration`'s before-image restore (a
|
||||
// transaction abort legitimately un-writes a vector the failed
|
||||
// transaction added). It is NOT right here: this fold replays
|
||||
// AFTER-IMAGES, and re-applying an already-intact record must
|
||||
// be byte-safe (this module's own invariant, see the log-authority
|
||||
// comment above) — silently erasing a landed vector because one
|
||||
// replayed fact's vector leg came back null is the exact defect
|
||||
// that left metadata-counted, never-enumerated rows in a
|
||||
// production store (confirmed root cause: the enumeration walk
|
||||
// used to key on the vector leg, so a preserved-but-then-deleted
|
||||
// vector made the row invisible while the ledger still counted
|
||||
// it by metadata). A genuine "unvector" has its own sanctioned,
|
||||
// ledger-correct path (`Brainy.unvectorNounForRootMigration`) —
|
||||
// never this raw primitive, and never the fold.
|
||||
const current =
|
||||
op.kind === 'verb'
|
||||
? await this.storage.readVerbRaw(op.id)
|
||||
: await this.storage.readNounRaw(op.id)
|
||||
image = { metadata: op.record.metadata, vector: current.vector ?? null }
|
||||
} else {
|
||||
image = { metadata: op.record.metadata, vector: op.record.vector }
|
||||
}
|
||||
if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
|
||||
else await this.storage.writeNounRaw(op.id, image)
|
||||
this.noteCheckpointDirty(op.kind, op.id)
|
||||
|
|
|
|||
|
|
@ -203,6 +203,40 @@ function idFromVectorPath(path: string): string {
|
|||
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Extract the entity id embedded in a metadata path
|
||||
* (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) — the IDENTITY-RECORD
|
||||
* mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their
|
||||
* population on this file (ADR-008 G1: the metadata record IS the population;
|
||||
* the vector leg is optional), so walk ordering and cursor resume derive the
|
||||
* id from THIS path, never the vector path — a row with metadata and no
|
||||
* vector file must still be listed, ordered, and resumable.
|
||||
* @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`).
|
||||
* @returns The entity id (the path segment immediately before `/metadata.json`).
|
||||
*/
|
||||
function idFromMetadataPath(path: string): string {
|
||||
const withoutSuffix = path.replace(/\/metadata\.json$/, '')
|
||||
const lastSlash = withoutSuffix.lastIndexOf('/')
|
||||
return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The sanctioned UNVECTORED shape for a noun hydrated during
|
||||
* enumeration when its identity record (metadata.json) exists but its vector
|
||||
* leg (vectors.json) does not — a fold-born metadata-only after-image, or any
|
||||
* row genuinely without a vector yet. Mirrors the shape
|
||||
* `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned
|
||||
* unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a
|
||||
* walk-yielded unvectored row is byte-shape-identical to one produced by that
|
||||
* migration. Callers already handle `vector: []` as first-class
|
||||
* (validateAddParams exempts it; index gates key on `length > 0`).
|
||||
* @param id - The noun id.
|
||||
* @returns A structurally-valid, vector-empty `HNSWNoun`.
|
||||
*/
|
||||
function unvectoredNoun(id: string): HNSWNoun {
|
||||
return { id, vector: [], connections: new Map<number, Set<string>>(), level: 0 }
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ID-first path for verb metadata
|
||||
* No type parameter needed - direct O(1) lookup by ID
|
||||
|
|
@ -1453,6 +1487,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* rollups are derived state with their own rebuild paths
|
||||
* (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`).
|
||||
*
|
||||
* EXACT-RESTORE PRIMITIVE — `vector: null` DELETES the vector leg, on
|
||||
* purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on
|
||||
* this to legitimately un-write a vector a failed transaction added. This
|
||||
* is deliberately NOT "preserve if absent" — a caller replaying an
|
||||
* AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must
|
||||
* apply preserve-if-absent itself BEFORE calling this, by reading the
|
||||
* current vector and carrying it forward when the after-image's own
|
||||
* vector leg is null/undefined but its metadata is not (see `replayFact`
|
||||
* for the implementation and full rationale). A caller that genuinely
|
||||
* wants to unvector a row uses the sanctioned, ledger-correct path
|
||||
* (`Brainy.unvectorNounForRootMigration`) — never this primitive.
|
||||
*
|
||||
* @param id - The entity id.
|
||||
* @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}.
|
||||
*/
|
||||
|
|
@ -1489,7 +1535,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
/**
|
||||
* Restore a relationship's raw stored objects byte-for-byte (verb-side
|
||||
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats).
|
||||
* mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats,
|
||||
* same EXACT-RESTORE contract — `vector: null` deletes, on purpose; the
|
||||
* fold's preserve-if-absent logic lives at its call site, not here).
|
||||
*
|
||||
* @param id - The relationship id.
|
||||
* @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}.
|
||||
|
|
@ -2183,9 +2231,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
// Stable within-shard order (by noun id) so offset windows and cursor resume
|
||||
// are deterministic; ids come from the path so skipped nouns are never read.
|
||||
//
|
||||
// IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record
|
||||
// (not the vector) IS the population — a noun with metadata and no vector
|
||||
// file (a fold-born after-image, see writeNounRaw's preserve-if-absent
|
||||
// contract) must still enumerate. Keying on metadata.json here means the
|
||||
// ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed)
|
||||
// and this walk agree on population by construction. Ordering is
|
||||
// unaffected for a healthy store: every vectored noun has both legs, so
|
||||
// the id set and sort order are identical to the old vectors.json keying.
|
||||
const entries = nounFiles
|
||||
.filter((p) => p.includes('/vectors.json'))
|
||||
.map((p) => ({ path: p, id: idFromVectorPath(p) }))
|
||||
.filter((p) => p.includes('/metadata.json'))
|
||||
.map((p) => ({ path: p, id: idFromMetadataPath(p) }))
|
||||
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
||||
|
||||
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
|
||||
|
|
@ -2210,13 +2267,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
) {
|
||||
const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
|
||||
const hydrated = await Promise.all(
|
||||
batch.map(async ({ path: nounPath }) => {
|
||||
batch.map(async ({ path: metadataPath, id }) => {
|
||||
try {
|
||||
const noun = await this.readCanonicalObject(nounPath)
|
||||
if (!noun) return null
|
||||
const deserialized = this.deserializeNoun(noun)
|
||||
const metadata = await this.getNounMetadata(deserialized.id)
|
||||
const metadata = await this.readCanonicalObject(metadataPath)
|
||||
if (!metadata) return null
|
||||
// The vector leg is OPTIONAL (population law): a metadata-only
|
||||
// row hydrates with the sanctioned unvectored shape rather than
|
||||
// being dropped from the walk. A fault reading the vector leg
|
||||
// is treated the same as absence — best-effort, matching the
|
||||
// canonical recount's tolerance for an unreadable vectors.json
|
||||
// (rebuildTypeCounts) — a vector-leg problem never hides an
|
||||
// otherwise-good identity record.
|
||||
let deserialized: HNSWNoun
|
||||
try {
|
||||
const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id))
|
||||
deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id)
|
||||
} catch {
|
||||
deserialized = unvectoredNoun(id)
|
||||
}
|
||||
return { deserialized, metadata }
|
||||
} catch (error) {
|
||||
// A TORN record must surface typed — a paginated read that
|
||||
|
|
@ -2226,7 +2294,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// walk's job is to HEAL PAST it — skip the victim, serve the rest.
|
||||
// Identity point-reads (get-by-id) still throw typed upstream.
|
||||
if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ }
|
||||
// Skip nouns that fail to load
|
||||
// Skip nouns whose IDENTITY record fails to load (the metadata
|
||||
// read above) — that is the one leg this walk cannot proceed
|
||||
// without.
|
||||
return null
|
||||
}
|
||||
})
|
||||
|
|
@ -2347,9 +2417,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
const shardDir = `entities/nouns/${shardHex}`
|
||||
try {
|
||||
const nounFiles = await this.listCanonicalObjects(shardDir)
|
||||
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching
|
||||
// comment in getNounsWithPagination: metadata.json is the population;
|
||||
// the vector leg is optional, so a metadata-only row must still be
|
||||
// listed (and here, for the unfiltered case, needs ZERO reads either
|
||||
// way — the id comes straight from the path).
|
||||
const entries = nounFiles
|
||||
.filter((p) => p.includes('/vectors.json'))
|
||||
.map((p) => idFromVectorPath(p))
|
||||
.filter((p) => p.includes('/metadata.json'))
|
||||
.map((p) => idFromMetadataPath(p))
|
||||
.sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
|
||||
const toWalk =
|
||||
cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
|
||||
|
|
@ -2560,23 +2635,79 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// Stable within-shard order (by verb id) so offset windows and cursor resume
|
||||
// are deterministic and consistent across calls. Ids come from the path, so
|
||||
// verbs skipped by the cursor are never read.
|
||||
//
|
||||
// IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of
|
||||
// this comment in getNounsWithPagination applies here too: metadata.json
|
||||
// is the population; keying on it here means this walk and the ledger
|
||||
// recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed)
|
||||
// agree on population by construction. Unchanged for a healthy store —
|
||||
// `relate()` always writes both legs of a verb in the same commit, so
|
||||
// the id set and order match the old vectors.json keying exactly; this
|
||||
// only additionally surfaces a fold-born metadata-only row (see
|
||||
// writeVerbRaw's preserve-if-absent contract).
|
||||
const entries = verbFiles
|
||||
.filter((p) => p.includes('/vectors.json'))
|
||||
.map((p) => ({ path: p, id: idFromVectorPath(p) }))
|
||||
.filter((p) => p.includes('/metadata.json'))
|
||||
.map((p) => ({ path: p, id: idFromMetadataPath(p) }))
|
||||
.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
||||
|
||||
for (const { path: verbPath, id: verbId } of entries) {
|
||||
for (const { path: metadataPath, id: verbId } of entries) {
|
||||
if (collected.length >= peekCount) break
|
||||
// Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
|
||||
// (later shards are processed in full). No read for skipped verbs.
|
||||
if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
|
||||
|
||||
try {
|
||||
const rawVerb = await this.readCanonicalObject(verbPath)
|
||||
if (!rawVerb) continue
|
||||
// Identity leg first — required. A verb this walk cannot read
|
||||
// metadata for cannot be hydrated at all (same as before).
|
||||
const metadata = await this.readCanonicalObject(metadataPath)
|
||||
if (!metadata) continue
|
||||
|
||||
// Deserialize connections Map from JSON storage format
|
||||
const verb = this.deserializeVerb(rawVerb)
|
||||
// The vector leg is the verb's STRUCTURAL core (verb/sourceId/
|
||||
// targetId live there — see coreTypes.ts HNSWVerb), unlike a
|
||||
// noun's vector, which is pure embedding data. `relate()` always
|
||||
// writes both legs atomically and verbs have no deferred-embed
|
||||
// path, so a healthy store's verbs always have both. A vector-leg
|
||||
// absence here can only be a fold-born after-image (see
|
||||
// writeVerbRaw's preserve-if-absent contract) — and unlike a
|
||||
// noun, this walk cannot safely FABRICATE sourceId/targetId to
|
||||
// synthesize a structurally-valid verb (an empty-string endpoint
|
||||
// would silently create a phantom edge — worse than omission).
|
||||
// If the metadata record happens to carry its own sourceId/
|
||||
// targetId (never true for current production writes, but not
|
||||
// disallowed — e.g. a future schema or a repair tool could
|
||||
// populate them), reconstruct from those; otherwise this row is
|
||||
// loudly skipped — counted by the ledger, but not returned as an
|
||||
// item, until a repair can supply the missing endpoints.
|
||||
const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId))
|
||||
let verb: HNSWVerb
|
||||
if (rawVerb) {
|
||||
verb = this.deserializeVerb(rawVerb)
|
||||
} else {
|
||||
const metaSourceId = (metadata as Record<string, unknown>).sourceId
|
||||
const metaTargetId = (metadata as Record<string, unknown>).targetId
|
||||
const metaVerbType = (metadata as Record<string, unknown>).verb
|
||||
if (
|
||||
typeof metaSourceId === 'string' && metaSourceId.length > 0 &&
|
||||
typeof metaTargetId === 'string' && metaTargetId.length > 0 &&
|
||||
typeof metaVerbType === 'string' && metaVerbType.length > 0
|
||||
) {
|
||||
verb = {
|
||||
id: verbId,
|
||||
vector: [],
|
||||
connections: new Map<number, Set<string>>(),
|
||||
verb: metaVerbType as VerbType,
|
||||
sourceId: metaSourceId,
|
||||
targetId: metaTargetId
|
||||
}
|
||||
} else {
|
||||
prodLog.error(
|
||||
`[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` +
|
||||
`record but no vector leg and no recoverable sourceId/targetId — ` +
|
||||
`skipping (counted by the ledger, not yielded; needs repair).`
|
||||
)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Apply type filter
|
||||
if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
|
||||
|
|
@ -2593,9 +2724,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
continue
|
||||
}
|
||||
|
||||
// Load metadata
|
||||
const metadata = await this.getVerbMetadata(verb.id)
|
||||
|
||||
// Apply subtype filter (requires metadata — checked AFTER load)
|
||||
if (filterSubtypes) {
|
||||
const subtype = metadata?.subtype as string | undefined
|
||||
|
|
|
|||
333
tests/integration/enumeration-population-law.test.ts
Normal file
333
tests/integration/enumeration-population-law.test.ts
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
/**
|
||||
* @module tests/integration/enumeration-population-law
|
||||
* @description THE POPULATION LAW (ADR-008 G1): the unfiltered noun/verb walk
|
||||
* and the canonical ALL scalar must agree on the population — a row's
|
||||
* IDENTITY RECORD (metadata.json) is what defines membership; the vector leg
|
||||
* is optional data, never a gate on visibility. Before this fix, the walk
|
||||
* (getNounsWithPagination / getNounIdsWithPagination / getVerbsWithPagination)
|
||||
* enumerated by keying on the VECTOR leg (`vectors.json`), so a row with
|
||||
* metadata and no vector file was counted by the ledger (already
|
||||
* metadata.json-keyed — see `rebuildTypeCounts`) but never yielded by the
|
||||
* walk: a permanent "counted but invisible" phantom for any downstream
|
||||
* consumer (a health-coverage row, an index-fill walk) that iterates the walk
|
||||
* to account for the ledger's total.
|
||||
*
|
||||
* Two legs are pinned here:
|
||||
* (a)/(b) LEG 1 — the walk re-keys on metadata.json. A fold-born
|
||||
* metadata-only row (the exact shape `GenerationStore.replayFact` can
|
||||
* leave behind, and the exact shape `writeNounRaw`/`writeVerbRaw` accept)
|
||||
* must be YIELDED, hydrated with the sanctioned unvectored shape
|
||||
* (`vector: []`) — not merely counted.
|
||||
*
|
||||
* For VERBS this closes only PARTIALLY: `sourceId`/`targetId` are
|
||||
* HNSWVerb's structural core and live ONLY in the vector leg (never in
|
||||
* metadata — see `RESERVED_RELATION_FIELDS` in reservedFields.ts, which
|
||||
* does not include them). A metadata-only verb row therefore cannot be
|
||||
* safely reconstructed without FABRICATING an edge's endpoints — which
|
||||
* would silently create a phantom relationship, strictly worse than the
|
||||
* original defect. The walk recovers the row when its metadata happens
|
||||
* to carry `sourceId`/`targetId` (a defensive, forward-compatible
|
||||
* fallback — never true for a CURRENT production write, but not
|
||||
* disallowed either); otherwise it counts the row (ledger, unchanged)
|
||||
* but loudly skips yielding it, logging the gap instead of hiding it.
|
||||
* Closing this fully requires persisting `sourceId`/`targetId` in verb
|
||||
* metadata — a schema change out of this task's scope; see the session
|
||||
* report for the explicit call-out.
|
||||
*
|
||||
* (c)/(d) LEG 2 — the recovery fold's preserve-if-absent contract, exercised
|
||||
* directly against `GenerationStore`/`FactLog` (below the `Brainy` API):
|
||||
* a metadata-only after-image replayed over an already-vectored row must
|
||||
* PRESERVE the existing vector leg (never delete it); a genuine tombstone
|
||||
* (both legs absent) still removes both legs.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
import { GenerationStore } from '../../src/db/generationStore.js'
|
||||
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
|
||||
import { LOG_AUTHORITY_PATH } from '../../src/db/logAuthority.js'
|
||||
import type { CommitFact } from '../../src/db/factLog.js'
|
||||
|
||||
describe('enumeration population law — LEG 1 (identity-keyed walk)', () => {
|
||||
let dir: string
|
||||
let brain: any
|
||||
|
||||
const open = async () => {
|
||||
const b: any = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
await b.init()
|
||||
return b
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-population-law-'))
|
||||
brain = await open()
|
||||
})
|
||||
afterEach(async () => {
|
||||
await brain.close?.().catch(() => {})
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('(a) nouns.all equals the unfiltered walk-yield count with a fold-born metadata-only row present', async () => {
|
||||
// Ordinary, fully-vectored background population.
|
||||
await brain.add({ data: 'one', type: 'document' })
|
||||
await brain.add({ data: 'two', type: 'document' })
|
||||
await brain.flush()
|
||||
|
||||
// THE EXACT PRE-FIX SHAPE: a metadata-only row against a FRESH id — no
|
||||
// vector ever existed for it. Written through the raw primitive directly,
|
||||
// exactly as `GenerationStore.replayFact` (the recovery fold) applies a
|
||||
// replayed after-image whose vector leg came back null.
|
||||
const freshId = randomUUID()
|
||||
await brain.storage.writeNounRaw(freshId, {
|
||||
metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 },
|
||||
vector: null
|
||||
})
|
||||
|
||||
// writeNounRaw bypasses count bookkeeping on purpose (its own JSDoc) — the
|
||||
// sanctioned recount brings the ledger scalar to ground truth. This walk
|
||||
// was ALREADY metadata.json-keyed before this fix (rebuildTypeCounts), so
|
||||
// the recount's answer does not depend on today's change.
|
||||
await brain.repairIndex()
|
||||
|
||||
const ledger = await brain.storage.getCanonicalCounts()
|
||||
const walk = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
|
||||
|
||||
expect(walk.items.length).toBe(ledger.nouns.all)
|
||||
expect(walk.totalCount).toBe(ledger.nouns.all)
|
||||
|
||||
const yielded = walk.items.find((n: any) => n.id === freshId)
|
||||
expect(yielded, 'the metadata-only row must be YIELDED, not merely counted').toBeDefined()
|
||||
expect(yielded.vector).toEqual([])
|
||||
})
|
||||
|
||||
it('(a-ids) getNounIdsWithPagination (the zero-read unfiltered enumerator) also yields the metadata-only row', async () => {
|
||||
await brain.add({ data: 'one', type: 'document' })
|
||||
await brain.flush()
|
||||
|
||||
const freshId = randomUUID()
|
||||
await brain.storage.writeNounRaw(freshId, {
|
||||
metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 },
|
||||
vector: null
|
||||
})
|
||||
await brain.repairIndex()
|
||||
|
||||
const ledger = await brain.storage.getCanonicalCounts()
|
||||
const page = await brain.storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
|
||||
expect(page.ids.length).toBe(ledger.nouns.all)
|
||||
expect(page.ids).toContain(freshId)
|
||||
})
|
||||
|
||||
it('(b) verbs.all counts a fold-born metadata-only row; the walk yields it when endpoints are recoverable from metadata, and loudly skips (never fabricates) when they are not', async () => {
|
||||
const a = await brain.add({ data: 'a', type: 'document' })
|
||||
const b = await brain.add({ data: 'b', type: 'document' })
|
||||
await brain.relate({ from: a, to: b, type: 'relatedTo' })
|
||||
await brain.flush()
|
||||
|
||||
// Case 1 — the REALISTIC production shape: metadata carries the verb
|
||||
// type (a reserved field, kept for backward compat) but never
|
||||
// sourceId/targetId — those are HNSWVerb's structural core and live
|
||||
// ONLY in the vector leg. The walk cannot safely fabricate them (an
|
||||
// empty-string endpoint would silently create a phantom edge), so this
|
||||
// row is counted by the ledger but not yielded — a documented,
|
||||
// loudly-logged gap, not a silent one.
|
||||
const gapId = randomUUID()
|
||||
await brain.storage.writeVerbRaw(gapId, {
|
||||
metadata: { verb: 'relatedTo', createdAt: Date.now(), updatedAt: Date.now(), weight: 1 },
|
||||
vector: null
|
||||
})
|
||||
|
||||
// Case 2 — endpoints ARE recoverable from metadata (never true for a
|
||||
// current production write; modeled here as what a repair tool or a
|
||||
// future schema could supply): the walk reconstructs and yields it.
|
||||
const recoveredId = randomUUID()
|
||||
await brain.storage.writeVerbRaw(recoveredId, {
|
||||
metadata: {
|
||||
verb: 'relatedTo',
|
||||
sourceId: a,
|
||||
targetId: b,
|
||||
createdAt: Date.now(),
|
||||
updatedAt: Date.now(),
|
||||
weight: 1
|
||||
},
|
||||
vector: null
|
||||
})
|
||||
|
||||
await brain.repairIndex()
|
||||
const ledger = await brain.storage.getCanonicalCounts()
|
||||
const walk = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } })
|
||||
|
||||
// The ledger counts every identity record — the real edge plus both
|
||||
// synthetic metadata-only rows — unaffected by whether the walk can
|
||||
// safely hydrate them.
|
||||
expect(ledger.verbs.all).toBe(3)
|
||||
|
||||
const recovered = walk.items.find((v: any) => v.id === recoveredId)
|
||||
expect(recovered, 'endpoints recoverable from metadata must be yielded').toBeDefined()
|
||||
expect(recovered.sourceId).toBe(a)
|
||||
expect(recovered.targetId).toBe(b)
|
||||
expect(recovered.vector).toEqual([])
|
||||
|
||||
// The documented gap: counted, not yielded — this is the one corner of
|
||||
// the population law this task does NOT close (see the session report).
|
||||
const gapped = walk.items.find((v: any) => v.id === gapId)
|
||||
expect(gapped).toBeUndefined()
|
||||
expect(walk.items.length).toBeLessThan(ledger.verbs.all)
|
||||
})
|
||||
})
|
||||
|
||||
describe('enumeration population law — LEG 2 (fold preserve-if-absent, below the Brainy API)', () => {
|
||||
/** A GenerationStore whose brain has already flipped to log authority — the
|
||||
* precondition for `replayFact` (the recovery fold) to run at open(). */
|
||||
async function openLogAuthorityStore(): Promise<{ storage: MemoryStorage; store: GenerationStore }> {
|
||||
const storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
await storage.writeRawObject(LOG_AUTHORITY_PATH, { authority: 'log' })
|
||||
const store = new GenerationStore(storage)
|
||||
await store.open()
|
||||
return { storage, store }
|
||||
}
|
||||
|
||||
it('(c) nouns: a metadata-only after-image replayed over a vectored row PRESERVES the vector; it stays readable and the vectored ledger is untouched either way', async () => {
|
||||
const { storage, store } = await openLogAuthorityStore()
|
||||
const id = randomUUID()
|
||||
const vectorRecord = { id, vector: [0.1, 0.2, 0.3], connections: {}, level: 0 }
|
||||
|
||||
// Generation 1 — a real, honest commit: both legs land together.
|
||||
await store.commitTransaction({
|
||||
touched: { nouns: [id], verbs: [] },
|
||||
execute: async () => {
|
||||
await storage.writeNounRaw(id, {
|
||||
metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 },
|
||||
vector: vectorRecord
|
||||
})
|
||||
}
|
||||
})
|
||||
const beforeVectoredCount = (await storage.getCanonicalCounts()).vectors.all
|
||||
|
||||
// THE ANOMALOUS FACT, crafted directly (bypassing commitTransaction,
|
||||
// whose honest read-after-write could never produce this on its own):
|
||||
// metadata changed, vector leg null, while the row is STILL vectored on
|
||||
// disk. This is exactly the shape the recovery fold must tolerate —
|
||||
// modeling the confirmed production defect at the replay boundary.
|
||||
const factLog = store.getFactLog()!
|
||||
const anomalousFact: CommitFact = {
|
||||
generation: 2,
|
||||
timestamp: Date.now(),
|
||||
ops: [
|
||||
{
|
||||
kind: 'noun',
|
||||
id,
|
||||
record: {
|
||||
metadata: { noun: 'document', createdAt: 1000, updatedAt: 2000, _rev: 2 },
|
||||
vector: null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
await factLog.append(anomalousFact)
|
||||
await factLog.sync()
|
||||
|
||||
// Reopen — a fresh GenerationStore over the SAME storage. Generation 2's
|
||||
// fact sits above the (still generation-1) manifest, so it replays
|
||||
// through the recovery fold — `replayFact`'s own call site.
|
||||
const store2 = new GenerationStore(storage)
|
||||
await store2.open()
|
||||
|
||||
const after = await storage.readNounRaw(id)
|
||||
expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull()
|
||||
expect((after.vector as { vector: number[] }).vector).toEqual([0.1, 0.2, 0.3])
|
||||
expect((after.metadata as { updatedAt: number }).updatedAt).toBe(2000) // the new metadata DID apply
|
||||
|
||||
// writeNounRaw bypasses ledger bookkeeping either way (by design — see
|
||||
// its JSDoc), so this scalar is unaffected by the replay regardless of
|
||||
// outcome; asserted for completeness against the task's exact wording.
|
||||
const afterVectoredCount = (await storage.getCanonicalCounts()).vectors.all
|
||||
expect(afterVectoredCount).toBe(beforeVectoredCount)
|
||||
})
|
||||
|
||||
it('(c-verb) verbs: a metadata-only after-image replayed over a vectored edge PRESERVES the vector leg (sourceId/targetId/verb intact)', async () => {
|
||||
const { storage, store } = await openLogAuthorityStore()
|
||||
const id = randomUUID()
|
||||
const sourceId = randomUUID()
|
||||
const targetId = randomUUID()
|
||||
const vectorRecord = { id, vector: [0.7, 0.8], connections: {}, verb: 'relatedTo', sourceId, targetId }
|
||||
|
||||
await store.commitTransaction({
|
||||
touched: { nouns: [], verbs: [id] },
|
||||
execute: async () => {
|
||||
await storage.writeVerbRaw(id, {
|
||||
metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 1000, weight: 1 },
|
||||
vector: vectorRecord
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
const factLog = store.getFactLog()!
|
||||
const anomalousFact: CommitFact = {
|
||||
generation: 2,
|
||||
timestamp: Date.now(),
|
||||
ops: [
|
||||
{
|
||||
kind: 'verb',
|
||||
id,
|
||||
record: {
|
||||
metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 2000, weight: 2 },
|
||||
vector: null
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
await factLog.append(anomalousFact)
|
||||
await factLog.sync()
|
||||
|
||||
const store2 = new GenerationStore(storage)
|
||||
await store2.open()
|
||||
|
||||
const after = await storage.readVerbRaw(id)
|
||||
expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull()
|
||||
expect((after.vector as { sourceId: string }).sourceId).toBe(sourceId)
|
||||
expect((after.vector as { targetId: string }).targetId).toBe(targetId)
|
||||
expect((after.metadata as { weight: number }).weight).toBe(2)
|
||||
})
|
||||
|
||||
it('(d) a genuine tombstone replay removes BOTH legs (never preserved)', async () => {
|
||||
const { storage, store } = await openLogAuthorityStore()
|
||||
const id = randomUUID()
|
||||
const vectorRecord = { id, vector: [0.4, 0.5, 0.6], connections: {}, level: 0 }
|
||||
|
||||
await store.commitTransaction({
|
||||
touched: { nouns: [id], verbs: [] },
|
||||
execute: async () => {
|
||||
await storage.writeNounRaw(id, {
|
||||
metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 },
|
||||
vector: vectorRecord
|
||||
})
|
||||
}
|
||||
})
|
||||
expect((await storage.readNounRaw(id)).vector).not.toBeNull() // sanity: it landed
|
||||
|
||||
const factLog = store.getFactLog()!
|
||||
await factLog.append({
|
||||
generation: 2,
|
||||
timestamp: Date.now(),
|
||||
ops: [{ kind: 'noun', id, record: null }] // a genuine tombstone — both legs absent
|
||||
})
|
||||
await factLog.sync()
|
||||
|
||||
const store2 = new GenerationStore(storage)
|
||||
await store2.open()
|
||||
|
||||
const after = await storage.readNounRaw(id)
|
||||
expect(after.metadata, 'a genuine delete removes the metadata leg').toBeNull()
|
||||
expect(after.vector, 'a genuine delete removes the vector leg too — preserve-if-absent never applies to a tombstone').toBeNull()
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue