fix(storage): enumeration re-keys on the identity record, not the vector leg

The noun/verb pagination walks (getNounsWithPagination,
getNounIdsWithPagination, getVerbsWithPagination) listed shard contents by
filtering for vectors.json, while the canonical count ledger has always
counted a row by its metadata.json presence alone. A row with metadata and
no vector file was therefore counted by the ledger but never yielded by the
walk — a permanent "counted but invisible" phantom for any downstream
consumer that iterates the walk to account for the ledger's total.

Nouns now enumerate by metadata.json and hydrate the vector leg optionally,
yielding the sanctioned unvectored shape (vector: []) when it's absent.
Verbs enumerate the same way, but a metadata-only verb row can only be fully
reconstructed when sourceId/targetId happen to be recoverable from metadata
(never true for a current production write — those fields live only in the
vector leg); otherwise the row is counted but loudly skipped rather than
fabricated, since a phantom edge with fake endpoints would be worse than the
original defect.

Separately, GenerationStore's recovery-fold replay (replayFact) now applies
preserve-if-absent: a metadata-only after-image replayed over an already-
vectored row carries the existing vector forward instead of deleting it via
writeNounRaw/writeVerbRaw's exact-restore null-means-delete contract (which
must stay exact for transaction-abort rollback). A genuine tombstone still
removes both legs.

src/db/generationStore.ts

@@ -654,10 +654,43 @@ export class GenerationStore {
         let replayed = 0
         const replayFact = async (fact: CommitFact): Promise<void> => {
           for (const op of fact.ops) {
-            const image =
-              op.record === null
-                ? { metadata: null, vector: null }
-                : { metadata: op.record.metadata, vector: op.record.vector }
+            let image: { metadata: unknown | null; vector: unknown | null }
+            if (op.record === null) {
+              // A genuine tombstone (both legs absent) — the fold removes
+              // both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw
+              // exact-restore contract.
+              image = { metadata: null, vector: null }
+            } else if (
+              op.record.metadata !== null &&
+              (op.record.vector === null || op.record.vector === undefined)
+            ) {
+              // PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's
+              // half): a metadata-only after-image must never DELETE an
+              // existing vector leg through the fold. `writeNounRaw`/
+              // `writeVerbRaw` are exact-restore primitives — a `vector:
+              // null` there means "delete", which is exactly right for
+              // `rollBackUncommittedGeneration`'s before-image restore (a
+              // transaction abort legitimately un-writes a vector the failed
+              // transaction added). It is NOT right here: this fold replays
+              // AFTER-IMAGES, and re-applying an already-intact record must
+              // be byte-safe (this module's own invariant, see the log-authority
+              // comment above) — silently erasing a landed vector because one
+              // replayed fact's vector leg came back null is the exact defect
+              // that left metadata-counted, never-enumerated rows in a
+              // production store (confirmed root cause: the enumeration walk
+              // used to key on the vector leg, so a preserved-but-then-deleted
+              // vector made the row invisible while the ledger still counted
+              // it by metadata). A genuine "unvector" has its own sanctioned,
+              // ledger-correct path (`Brainy.unvectorNounForRootMigration`) —
+              // never this raw primitive, and never the fold.
+              const current =
+                op.kind === 'verb'
+                  ? await this.storage.readVerbRaw(op.id)
+                  : await this.storage.readNounRaw(op.id)
+              image = { metadata: op.record.metadata, vector: current.vector ?? null }
+            } else {
+              image = { metadata: op.record.metadata, vector: op.record.vector }
+            }
             if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image)
             else await this.storage.writeNounRaw(op.id, image)
             this.noteCheckpointDirty(op.kind, op.id)

src/storage/baseStorage.ts

@@ -203,6 +203,40 @@ function idFromVectorPath(path: string): string {
   return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
 }
 
+/**
+ * @description Extract the entity id embedded in a metadata path
+ * (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) — the IDENTITY-RECORD
+ * mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their
+ * population on this file (ADR-008 G1: the metadata record IS the population;
+ * the vector leg is optional), so walk ordering and cursor resume derive the
+ * id from THIS path, never the vector path — a row with metadata and no
+ * vector file must still be listed, ordered, and resumable.
+ * @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`).
+ * @returns The entity id (the path segment immediately before `/metadata.json`).
+ */
+function idFromMetadataPath(path: string): string {
+  const withoutSuffix = path.replace(/\/metadata\.json$/, '')
+  const lastSlash = withoutSuffix.lastIndexOf('/')
+  return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix
+}
+
+/**
+ * @description The sanctioned UNVECTORED shape for a noun hydrated during
+ * enumeration when its identity record (metadata.json) exists but its vector
+ * leg (vectors.json) does not — a fold-born metadata-only after-image, or any
+ * row genuinely without a vector yet. Mirrors the shape
+ * `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned
+ * unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a
+ * walk-yielded unvectored row is byte-shape-identical to one produced by that
+ * migration. Callers already handle `vector: []` as first-class
+ * (validateAddParams exempts it; index gates key on `length > 0`).
+ * @param id - The noun id.
+ * @returns A structurally-valid, vector-empty `HNSWNoun`.
+ */
+function unvectoredNoun(id: string): HNSWNoun {
+  return { id, vector: [], connections: new Map<number, Set<string>>(), level: 0 }
+}
+
 /**
  * Get ID-first path for verb metadata
  * No type parameter needed - direct O(1) lookup by ID
@@ -1453,6 +1487,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
    * rollups are derived state with their own rebuild paths
    * (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`).
    *
+   * EXACT-RESTORE PRIMITIVE — `vector: null` DELETES the vector leg, on
+   * purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on
+   * this to legitimately un-write a vector a failed transaction added. This
+   * is deliberately NOT "preserve if absent" — a caller replaying an
+   * AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must
+   * apply preserve-if-absent itself BEFORE calling this, by reading the
+   * current vector and carrying it forward when the after-image's own
+   * vector leg is null/undefined but its metadata is not (see `replayFact`
+   * for the implementation and full rationale). A caller that genuinely
+   * wants to unvector a row uses the sanctioned, ledger-correct path
+   * (`Brainy.unvectorNounForRootMigration`) — never this primitive.
+   *
    * @param id - The entity id.
    * @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}.
    */
@@ -1489,7 +1535,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
   /**
    * Restore a relationship's raw stored objects byte-for-byte (verb-side
-   * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats).
+   * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats,
+   * same EXACT-RESTORE contract — `vector: null` deletes, on purpose; the
+   * fold's preserve-if-absent logic lives at its call site, not here).
    *
    * @param id - The relationship id.
    * @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}.
@@ -2183,9 +2231,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
         // Stable within-shard order (by noun id) so offset windows and cursor resume
         // are deterministic; ids come from the path so skipped nouns are never read.
+        //
+        // IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record
+        // (not the vector) IS the population — a noun with metadata and no vector
+        // file (a fold-born after-image, see writeNounRaw's preserve-if-absent
+        // contract) must still enumerate. Keying on metadata.json here means the
+        // ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed)
+        // and this walk agree on population by construction. Ordering is
+        // unaffected for a healthy store: every vectored noun has both legs, so
+        // the id set and sort order are identical to the old vectors.json keying.
         const entries = nounFiles
-          .filter((p) => p.includes('/vectors.json'))
-          .map((p) => ({ path: p, id: idFromVectorPath(p) }))
+          .filter((p) => p.includes('/metadata.json'))
+          .map((p) => ({ path: p, id: idFromMetadataPath(p) }))
           .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
 
         // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor
@@ -2210,13 +2267,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
         ) {
           const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY)
           const hydrated = await Promise.all(
-            batch.map(async ({ path: nounPath }) => {
+            batch.map(async ({ path: metadataPath, id }) => {
               try {
-                const noun = await this.readCanonicalObject(nounPath)
-                if (!noun) return null
-                const deserialized = this.deserializeNoun(noun)
-                const metadata = await this.getNounMetadata(deserialized.id)
+                const metadata = await this.readCanonicalObject(metadataPath)
                 if (!metadata) return null
+                // The vector leg is OPTIONAL (population law): a metadata-only
+                // row hydrates with the sanctioned unvectored shape rather than
+                // being dropped from the walk. A fault reading the vector leg
+                // is treated the same as absence — best-effort, matching the
+                // canonical recount's tolerance for an unreadable vectors.json
+                // (rebuildTypeCounts) — a vector-leg problem never hides an
+                // otherwise-good identity record.
+                let deserialized: HNSWNoun
+                try {
+                  const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id))
+                  deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id)
+                } catch {
+                  deserialized = unvectoredNoun(id)
+                }
                 return { deserialized, metadata }
               } catch (error) {
                 // A TORN record must surface typed — a paginated read that
@@ -2226,7 +2294,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
                 // walk's job is to HEAL PAST it — skip the victim, serve the rest.
                 // Identity point-reads (get-by-id) still throw typed upstream.
                 if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ }
-                // Skip nouns that fail to load
+                // Skip nouns whose IDENTITY record fails to load (the metadata
+                // read above) — that is the one leg this walk cannot proceed
+                // without.
                 return null
               }
             })
@@ -2347,9 +2417,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
       const shardDir = `entities/nouns/${shardHex}`
       try {
         const nounFiles = await this.listCanonicalObjects(shardDir)
+        // IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching
+        // comment in getNounsWithPagination: metadata.json is the population;
+        // the vector leg is optional, so a metadata-only row must still be
+        // listed (and here, for the unfiltered case, needs ZERO reads either
+        // way — the id comes straight from the path).
         const entries = nounFiles
-          .filter((p) => p.includes('/vectors.json'))
-          .map((p) => idFromVectorPath(p))
+          .filter((p) => p.includes('/metadata.json'))
+          .map((p) => idFromMetadataPath(p))
           .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
         const toWalk =
           cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries
@@ -2560,23 +2635,79 @@ export abstract class BaseStorage extends BaseStorageAdapter {
         // Stable within-shard order (by verb id) so offset windows and cursor resume
         // are deterministic and consistent across calls. Ids come from the path, so
         // verbs skipped by the cursor are never read.
+        //
+        // IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of
+        // this comment in getNounsWithPagination applies here too: metadata.json
+        // is the population; keying on it here means this walk and the ledger
+        // recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed)
+        // agree on population by construction. Unchanged for a healthy store —
+        // `relate()` always writes both legs of a verb in the same commit, so
+        // the id set and order match the old vectors.json keying exactly; this
+        // only additionally surfaces a fold-born metadata-only row (see
+        // writeVerbRaw's preserve-if-absent contract).
         const entries = verbFiles
-          .filter((p) => p.includes('/vectors.json'))
-          .map((p) => ({ path: p, id: idFromVectorPath(p) }))
+          .filter((p) => p.includes('/metadata.json'))
+          .map((p) => ({ path: p, id: idFromMetadataPath(p) }))
           .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
 
-        for (const { path: verbPath, id: verbId } of entries) {
+        for (const { path: metadataPath, id: verbId } of entries) {
           if (collected.length >= peekCount) break
           // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id
           // (later shards are processed in full). No read for skipped verbs.
           if (cursor && shard === cursor.shard && verbId <= cursor.id) continue
 
           try {
-            const rawVerb = await this.readCanonicalObject(verbPath)
-            if (!rawVerb) continue
+            // Identity leg first — required. A verb this walk cannot read
+            // metadata for cannot be hydrated at all (same as before).
+            const metadata = await this.readCanonicalObject(metadataPath)
+            if (!metadata) continue
 
-            // Deserialize connections Map from JSON storage format
-            const verb = this.deserializeVerb(rawVerb)
+            // The vector leg is the verb's STRUCTURAL core (verb/sourceId/
+            // targetId live there — see coreTypes.ts HNSWVerb), unlike a
+            // noun's vector, which is pure embedding data. `relate()` always
+            // writes both legs atomically and verbs have no deferred-embed
+            // path, so a healthy store's verbs always have both. A vector-leg
+            // absence here can only be a fold-born after-image (see
+            // writeVerbRaw's preserve-if-absent contract) — and unlike a
+            // noun, this walk cannot safely FABRICATE sourceId/targetId to
+            // synthesize a structurally-valid verb (an empty-string endpoint
+            // would silently create a phantom edge — worse than omission).
+            // If the metadata record happens to carry its own sourceId/
+            // targetId (never true for current production writes, but not
+            // disallowed — e.g. a future schema or a repair tool could
+            // populate them), reconstruct from those; otherwise this row is
+            // loudly skipped — counted by the ledger, but not returned as an
+            // item, until a repair can supply the missing endpoints.
+            const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId))
+            let verb: HNSWVerb
+            if (rawVerb) {
+              verb = this.deserializeVerb(rawVerb)
+            } else {
+              const metaSourceId = (metadata as Record<string, unknown>).sourceId
+              const metaTargetId = (metadata as Record<string, unknown>).targetId
+              const metaVerbType = (metadata as Record<string, unknown>).verb
+              if (
+                typeof metaSourceId === 'string' && metaSourceId.length > 0 &&
+                typeof metaTargetId === 'string' && metaTargetId.length > 0 &&
+                typeof metaVerbType === 'string' && metaVerbType.length > 0
+              ) {
+                verb = {
+                  id: verbId,
+                  vector: [],
+                  connections: new Map<number, Set<string>>(),
+                  verb: metaVerbType as VerbType,
+                  sourceId: metaSourceId,
+                  targetId: metaTargetId
+                }
+              } else {
+                prodLog.error(
+                  `[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` +
+                    `record but no vector leg and no recoverable sourceId/targetId — ` +
+                    `skipping (counted by the ledger, not yielded; needs repair).`
+                )
+                continue
+              }
+            }
 
             // Apply type filter
             if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) {
@@ -2593,9 +2724,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
               continue
             }
 
-            // Load metadata
-            const metadata = await this.getVerbMetadata(verb.id)
-
             // Apply subtype filter (requires metadata — checked AFTER load)
             if (filterSubtypes) {
               const subtype = metadata?.subtype as string | undefined