open-brainy/tests/integration/enumeration-population-law.test.ts

/**
 * @module tests/integration/enumeration-population-law
 * @description THE POPULATION LAW (ADR-008 G1): the unfiltered noun/verb walk
 * and the canonical ALL scalar must agree on the population — a row's
 * IDENTITY RECORD (metadata.json) is what defines membership; the vector leg
 * is optional data, never a gate on visibility. Before this fix, the walk
 * (getNounsWithPagination / getNounIdsWithPagination / getVerbsWithPagination)
 * enumerated by keying on the VECTOR leg (`vectors.json`), so a row with
 * metadata and no vector file was counted by the ledger (already
 * metadata.json-keyed — see `rebuildTypeCounts`) but never yielded by the
 * walk: a permanent "counted but invisible" phantom for any downstream
 * consumer (a health-coverage row, an index-fill walk) that iterates the walk
 * to account for the ledger's total.
 *
 * Two legs are pinned here:
 * (a)/(b) LEG 1 — the walk re-keys on metadata.json. A fold-born
 *     metadata-only row (the exact shape `GenerationStore.replayFact` can
 *     leave behind, and the exact shape `writeNounRaw`/`writeVerbRaw` accept)
 *     must be YIELDED, hydrated with the sanctioned unvectored shape
 *     (`vector: []`) — not merely counted.
 *
 *     For VERBS this closes only PARTIALLY: `sourceId`/`targetId` are
 *     HNSWVerb's structural core and live ONLY in the vector leg (never in
 *     metadata — see `RESERVED_RELATION_FIELDS` in reservedFields.ts, which
 *     does not include them). A metadata-only verb row therefore cannot be
 *     safely reconstructed without FABRICATING an edge's endpoints — which
 *     would silently create a phantom relationship, strictly worse than the
 *     original defect. The walk recovers the row when its metadata happens
 *     to carry `sourceId`/`targetId` (a defensive, forward-compatible
 *     fallback — never true for a CURRENT production write, but not
 *     disallowed either); otherwise it counts the row (ledger, unchanged)
 *     but loudly skips yielding it, logging the gap instead of hiding it.
 *     Closing this fully requires persisting `sourceId`/`targetId` in verb
 *     metadata — a schema change out of this task's scope; see the session
 *     report for the explicit call-out.
 *
 * (c)/(d) LEG 2 — the recovery fold's preserve-if-absent contract, exercised
 *     directly against `GenerationStore`/`FactLog` (below the `Brainy` API):
 *     a metadata-only after-image replayed over an already-vectored row must
 *     PRESERVE the existing vector leg (never delete it); a genuine tombstone
 *     (both legs absent) still removes both legs.
 */
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { randomUUID } from 'node:crypto'
import { Brainy } from '../../src/index.js'
import { GenerationStore } from '../../src/db/generationStore.js'
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
import { LOG_AUTHORITY_PATH } from '../../src/db/logAuthority.js'
import type { CommitFact } from '../../src/db/factLog.js'

describe('enumeration population law — LEG 1 (identity-keyed walk)', () => {
  let dir: string
  let brain: any

  const open = async () => {
    const b: any = new Brainy({
      requireSubtype: false,
      storage: { type: 'filesystem', path: dir },
      silent: true,
      dimensions: 384
    })
    await b.init()
    return b
  }

  beforeEach(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-population-law-'))
    brain = await open()
  })
  afterEach(async () => {
    await brain.close?.().catch(() => {})
    fs.rmSync(dir, { recursive: true, force: true })
  })

  it('(a) nouns.all equals the unfiltered walk-yield count with a fold-born metadata-only row present', async () => {
    // Ordinary, fully-vectored background population.
    await brain.add({ data: 'one', type: 'document' })
    await brain.add({ data: 'two', type: 'document' })
    await brain.flush()

    // THE EXACT PRE-FIX SHAPE: a metadata-only row against a FRESH id — no
    // vector ever existed for it. Written through the raw primitive directly,
    // exactly as `GenerationStore.replayFact` (the recovery fold) applies a
    // replayed after-image whose vector leg came back null.
    const freshId = randomUUID()
    await brain.storage.writeNounRaw(freshId, {
      metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 },
      vector: null
    })

    // writeNounRaw bypasses count bookkeeping on purpose (its own JSDoc) — the
    // sanctioned recount brings the ledger scalar to ground truth. This walk
    // was ALREADY metadata.json-keyed before this fix (rebuildTypeCounts), so
    // the recount's answer does not depend on today's change.
    await brain.repairIndex()

    const ledger = await brain.storage.getCanonicalCounts()
    const walk = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })

    expect(walk.items.length).toBe(ledger.nouns.all)
    expect(walk.totalCount).toBe(ledger.nouns.all)

    const yielded = walk.items.find((n: any) => n.id === freshId)
    expect(yielded, 'the metadata-only row must be YIELDED, not merely counted').toBeDefined()
    expect(yielded.vector).toEqual([])
  })

  it('(a-ids) getNounIdsWithPagination (the zero-read unfiltered enumerator) also yields the metadata-only row', async () => {
    await brain.add({ data: 'one', type: 'document' })
    await brain.flush()

    const freshId = randomUUID()
    await brain.storage.writeNounRaw(freshId, {
      metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 },
      vector: null
    })
    await brain.repairIndex()

    const ledger = await brain.storage.getCanonicalCounts()
    const page = await brain.storage.getNounIdsWithPagination({ limit: 1000, offset: 0 })
    expect(page.ids.length).toBe(ledger.nouns.all)
    expect(page.ids).toContain(freshId)
  })

  it('(b) verbs.all counts a fold-born metadata-only row; the walk yields it when endpoints are recoverable from metadata, and loudly skips (never fabricates) when they are not', async () => {
    const a = await brain.add({ data: 'a', type: 'document' })
    const b = await brain.add({ data: 'b', type: 'document' })
    await brain.relate({ from: a, to: b, type: 'relatedTo' })
    await brain.flush()

    // Case 1 — the REALISTIC production shape: metadata carries the verb
    // type (a reserved field, kept for backward compat) but never
    // sourceId/targetId — those are HNSWVerb's structural core and live
    // ONLY in the vector leg. The walk cannot safely fabricate them (an
    // empty-string endpoint would silently create a phantom edge), so this
    // row is counted by the ledger but not yielded — a documented,
    // loudly-logged gap, not a silent one.
    const gapId = randomUUID()
    await brain.storage.writeVerbRaw(gapId, {
      metadata: { verb: 'relatedTo', createdAt: Date.now(), updatedAt: Date.now(), weight: 1 },
      vector: null
    })

    // Case 2 — endpoints ARE recoverable from metadata (never true for a
    // current production write; modeled here as what a repair tool or a
    // future schema could supply): the walk reconstructs and yields it.
    const recoveredId = randomUUID()
    await brain.storage.writeVerbRaw(recoveredId, {
      metadata: {
        verb: 'relatedTo',
        sourceId: a,
        targetId: b,
        createdAt: Date.now(),
        updatedAt: Date.now(),
        weight: 1
      },
      vector: null
    })

    await brain.repairIndex()
    const ledger = await brain.storage.getCanonicalCounts()
    const walk = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } })

    // The ledger counts every identity record — the real edge plus both
    // synthetic metadata-only rows — unaffected by whether the walk can
    // safely hydrate them.
    expect(ledger.verbs.all).toBe(3)

    const recovered = walk.items.find((v: any) => v.id === recoveredId)
    expect(recovered, 'endpoints recoverable from metadata must be yielded').toBeDefined()
    expect(recovered.sourceId).toBe(a)
    expect(recovered.targetId).toBe(b)
    expect(recovered.vector).toEqual([])

    // The documented gap: counted, not yielded — this is the one corner of
    // the population law this task does NOT close (see the session report).
    const gapped = walk.items.find((v: any) => v.id === gapId)
    expect(gapped).toBeUndefined()
    expect(walk.items.length).toBeLessThan(ledger.verbs.all)
  })
})

describe('enumeration population law — LEG 2 (fold preserve-if-absent, below the Brainy API)', () => {
  /** A GenerationStore whose brain has already flipped to log authority — the
   *  precondition for `replayFact` (the recovery fold) to run at open(). */
  async function openLogAuthorityStore(): Promise<{ storage: MemoryStorage; store: GenerationStore }> {
    const storage = new MemoryStorage()
    await storage.init()
    await storage.writeRawObject(LOG_AUTHORITY_PATH, { authority: 'log' })
    const store = new GenerationStore(storage)
    await store.open()
    return { storage, store }
  }

  it('(c) nouns: a metadata-only after-image replayed over a vectored row PRESERVES the vector; it stays readable and the vectored ledger is untouched either way', async () => {
    const { storage, store } = await openLogAuthorityStore()
    const id = randomUUID()
    const vectorRecord = { id, vector: [0.1, 0.2, 0.3], connections: {}, level: 0 }

    // Generation 1 — a real, honest commit: both legs land together.
    await store.commitTransaction({
      touched: { nouns: [id], verbs: [] },
      execute: async () => {
        await storage.writeNounRaw(id, {
          metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 },
          vector: vectorRecord
        })
      }
    })
    const beforeVectoredCount = (await storage.getCanonicalCounts()).vectors.all

    // THE ANOMALOUS FACT, crafted directly (bypassing commitTransaction,
    // whose honest read-after-write could never produce this on its own):
    // metadata changed, vector leg null, while the row is STILL vectored on
    // disk. This is exactly the shape the recovery fold must tolerate —
    // modeling the confirmed production defect at the replay boundary.
    const factLog = store.getFactLog()!
    const anomalousFact: CommitFact = {
      generation: 2,
      timestamp: Date.now(),
      ops: [
        {
          kind: 'noun',
          id,
          record: {
            metadata: { noun: 'document', createdAt: 1000, updatedAt: 2000, _rev: 2 },
            vector: null
          }
        }
      ]
    }
    await factLog.append(anomalousFact)
    await factLog.sync()

    // Reopen — a fresh GenerationStore over the SAME storage. Generation 2's
    // fact sits above the (still generation-1) manifest, so it replays
    // through the recovery fold — `replayFact`'s own call site.
    const store2 = new GenerationStore(storage)
    await store2.open()

    const after = await storage.readNounRaw(id)
    expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull()
    expect((after.vector as { vector: number[] }).vector).toEqual([0.1, 0.2, 0.3])
    expect((after.metadata as { updatedAt: number }).updatedAt).toBe(2000) // the new metadata DID apply

    // writeNounRaw bypasses ledger bookkeeping either way (by design — see
    // its JSDoc), so this scalar is unaffected by the replay regardless of
    // outcome; asserted for completeness against the task's exact wording.
    const afterVectoredCount = (await storage.getCanonicalCounts()).vectors.all
    expect(afterVectoredCount).toBe(beforeVectoredCount)
  })

  it('(c-verb) verbs: a metadata-only after-image replayed over a vectored edge PRESERVES the vector leg (sourceId/targetId/verb intact)', async () => {
    const { storage, store } = await openLogAuthorityStore()
    const id = randomUUID()
    const sourceId = randomUUID()
    const targetId = randomUUID()
    const vectorRecord = { id, vector: [0.7, 0.8], connections: {}, verb: 'relatedTo', sourceId, targetId }

    await store.commitTransaction({
      touched: { nouns: [], verbs: [id] },
      execute: async () => {
        await storage.writeVerbRaw(id, {
          metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 1000, weight: 1 },
          vector: vectorRecord
        })
      }
    })

    const factLog = store.getFactLog()!
    const anomalousFact: CommitFact = {
      generation: 2,
      timestamp: Date.now(),
      ops: [
        {
          kind: 'verb',
          id,
          record: {
            metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 2000, weight: 2 },
            vector: null
          }
        }
      ]
    }
    await factLog.append(anomalousFact)
    await factLog.sync()

    const store2 = new GenerationStore(storage)
    await store2.open()

    const after = await storage.readVerbRaw(id)
    expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull()
    expect((after.vector as { sourceId: string }).sourceId).toBe(sourceId)
    expect((after.vector as { targetId: string }).targetId).toBe(targetId)
    expect((after.metadata as { weight: number }).weight).toBe(2)
  })

  it('(d) a genuine tombstone replay removes BOTH legs (never preserved)', async () => {
    const { storage, store } = await openLogAuthorityStore()
    const id = randomUUID()
    const vectorRecord = { id, vector: [0.4, 0.5, 0.6], connections: {}, level: 0 }

    await store.commitTransaction({
      touched: { nouns: [id], verbs: [] },
      execute: async () => {
        await storage.writeNounRaw(id, {
          metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 },
          vector: vectorRecord
        })
      }
    })
    expect((await storage.readNounRaw(id)).vector).not.toBeNull() // sanity: it landed

    const factLog = store.getFactLog()!
    await factLog.append({
      generation: 2,
      timestamp: Date.now(),
      ops: [{ kind: 'noun', id, record: null }] // a genuine tombstone — both legs absent
    })
    await factLog.sync()

    const store2 = new GenerationStore(storage)
    await store2.open()

    const after = await storage.readNounRaw(id)
    expect(after.metadata, 'a genuine delete removes the metadata leg').toBeNull()
    expect(after.vector, 'a genuine delete removes the vector leg too — preserve-if-absent never applies to a tombstone').toBeNull()
  })
})