Merge branch 'release/8.11.0'

tests/unit/db/db-portable-graph.test.ts

@@ -9,7 +9,7 @@
  * subtype-required default.
  */
 
-import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
 import { randomUUID } from 'node:crypto'
 import * as fs from 'node:fs/promises'
 import * as os from 'node:os'
@@ -19,6 +19,7 @@ import { createTestConfig } from '../../helpers/test-factory'
 import { NounType, VerbType } from '../../../src/types/graphTypes'
 import { validatePortableGraph } from '../../../src/db/portableGraph'
 import type { PortableGraph } from '../../../src/db/portableGraph'
+import { CanonicalEnumerationUnavailableError } from '../../../src/db/errors'
 
 describe('8.0 portable graph export/import (PortableGraph v1)', () => {
   let brain: Brainy
@@ -293,3 +294,257 @@ describe('8.0 export includeContent (VFS blobs, filesystem)', () => {
     }
   })
 })
+
+describe('8.0 export enumeration:"canonical" — canon-complete against index blindness', () => {
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy(createTestConfig())
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('(i) equals the index-based export when the index is healthy — same entity ids, relations, vectors', async () => {
+    const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' })
+    const b = await brain.add({ data: 'Bob', type: NounType.Person, subtype: 'employee' })
+    const c = await brain.add({ data: 'Acme', type: NounType.Organization, subtype: 'vendor' })
+    await brain.relate({ from: a, to: b, type: VerbType.FriendOf, subtype: 'close' })
+    await brain.relate({ from: a, to: c, type: VerbType.WorksWith, subtype: 'full-time' })
+
+    const indexExport = await brain.export({}, { includeVectors: true, enumeration: 'index' })
+    const canonicalExport = await brain.export({}, { includeVectors: true, enumeration: 'canonical' })
+
+    expect(canonicalExport.entities.map((e) => e.id).sort()).toEqual(
+      indexExport.entities.map((e) => e.id).sort()
+    )
+    expect(canonicalExport.relations.map((r) => r.id).sort()).toEqual(
+      indexExport.relations.map((r) => r.id).sort()
+    )
+    expect(canonicalExport.entities.map((e) => e.id).sort()).toEqual([a, b, c].sort())
+    for (const e of canonicalExport.entities) {
+      expect(e.vector?.length).toBeGreaterThan(0)
+    }
+    expect(canonicalExport.drift).toBeUndefined() // reportIndexDrift not requested
+  })
+
+  it('(ii) survives simulated metadata-index blindness; the index export misses the record; drift names it canonicalOnly', async () => {
+    const staff = await brain.add({
+      data: 'Staff',
+      type: NounType.Person,
+      subtype: 'employee',
+      metadata: { role: 'staff' }
+    })
+    const other = await brain.add({
+      data: 'Other',
+      type: NounType.Person,
+      subtype: 'employee',
+      metadata: { role: 'staff' }
+    })
+
+    // Surgically poison the metadata index (the lowest-level seam the existing
+    // find() phantom-row guard tests use — see find-index-integrity-guard.test.ts,
+    // which does the mirror-image ADD case) so the predicate query
+    // enumeration:'index' issues (find({ type: Person })) never returns `staff` —
+    // a real canonical record the index has lost track of, the exact
+    // canon-present/index-missing state canonical mode exists to survive.
+    const mi = (brain as any).metadataIndex
+    const original = mi.getIdsForFilter.bind(mi)
+    mi.getIdsForFilter = async (filter: any, opts?: any): Promise<string[]> => {
+      const ids: string[] = await original(filter, opts)
+      return ids.filter((id: string) => id !== staff)
+    }
+
+    try {
+      const indexExport = await brain.export({ type: NounType.Person }, { enumeration: 'index' })
+      expect(indexExport.entities.map((e) => e.id)).not.toContain(staff)
+      expect(indexExport.entities.map((e) => e.id)).toContain(other)
+
+      const canonicalExport = await brain.export(
+        { type: NounType.Person },
+        { enumeration: 'canonical', reportIndexDrift: true }
+      )
+      expect(canonicalExport.entities.map((e) => e.id)).toContain(staff)
+      expect(canonicalExport.entities.map((e) => e.id)).toContain(other)
+      expect(canonicalExport.drift?.canonicalOnly).toEqual([staff])
+      expect(canonicalExport.drift?.indexOnly).toEqual([])
+    } finally {
+      mi.getIdsForFilter = original
+    }
+  })
+
+  it('(iii) drift report shape + loud console.warn only when nonzero', async () => {
+    const staff = await brain.add({ data: 'Staff', type: NounType.Person, subtype: 'employee' })
+    await brain.add({ data: 'Other', type: NounType.Person, subtype: 'employee' })
+
+    const mi = (brain as any).metadataIndex
+    const original = mi.getIdsForFilter.bind(mi)
+    mi.getIdsForFilter = async (filter: any, opts?: any): Promise<string[]> => {
+      const ids: string[] = await original(filter, opts)
+      return ids.filter((id: string) => id !== staff)
+    }
+
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
+    try {
+      const drifted = await brain.export(
+        { type: NounType.Person },
+        { enumeration: 'canonical', reportIndexDrift: true }
+      )
+      expect(drifted.drift).toEqual({ canonicalOnly: [staff], indexOnly: [] })
+      expect(warnSpy).toHaveBeenCalledTimes(1)
+      expect(warnSpy.mock.calls[0].join(' ')).toMatch(/drift/i)
+    } finally {
+      mi.getIdsForFilter = original
+      warnSpy.mockClear()
+    }
+
+    // Healthy index: drift is reported (both lists present) but never warned about.
+    try {
+      const healthy = await brain.export(
+        { type: NounType.Person },
+        { enumeration: 'canonical', reportIndexDrift: true }
+      )
+      expect(healthy.drift).toEqual({ canonicalOnly: [], indexOnly: [] })
+      expect(warnSpy).not.toHaveBeenCalled()
+    } finally {
+      warnSpy.mockRestore()
+    }
+  })
+
+  it('(iv) throws CanonicalEnumerationUnavailableError on a historical asOf() view and a speculative with() overlay', async () => {
+    const a = '22222222-2222-4222-8222-222222222222'
+    const b = '33333333-3333-4333-8333-333333333333'
+    await brain.transact([{ op: 'add', id: a, data: 'First', type: NounType.Thing, subtype: 'x' }])
+    const g1 = brain.generation()
+    await brain.transact([{ op: 'add', id: b, data: 'Second', type: NounType.Thing, subtype: 'x' }])
+
+    const past = await brain.asOf(g1)
+    try {
+      await expect(past.export({}, { enumeration: 'canonical' })).rejects.toThrow(
+        CanonicalEnumerationUnavailableError
+      )
+      // The default (index) mode is unaffected — still a valid time-travel export.
+      const backup = await past.export()
+      expect(backup.entities.map((e) => e.id)).toContain(a)
+    } finally {
+      await past.release()
+    }
+
+    const speculativeId = '11111111-1111-4111-8111-111111111111'
+    const view = await brain.now().with([
+      { op: 'add', id: speculativeId, data: 'Speculative', type: NounType.Thing, subtype: 'x' }
+    ])
+    try {
+      await expect(view.export({}, { enumeration: 'canonical' })).rejects.toThrow(
+        CanonicalEnumerationUnavailableError
+      )
+    } finally {
+      await view.release()
+    }
+  })
+
+  it('throws a plain Error when enumeration:"canonical" has no storage adapter to walk', async () => {
+    const { exportGraph } = await import('../../../src/db/portableGraph')
+    const readerOnly = { get: async () => null, find: async () => [], related: async () => [] }
+    await expect(
+      exportGraph(readerOnly as any, undefined, {}, { enumeration: 'canonical' })
+    ).rejects.toThrow(/enumeration:'canonical' requires a storage adapter/)
+  })
+})
+
+describe('8.0 export includeHidden — every visibility tier for migration-grade canon completeness', () => {
+  // The fixed-id VFS root Brainy.init() always creates is the one 'system'-visibility
+  // entity a consumer can rely on existing (visibility:'system' is not settable via the
+  // public add() API — "intentionally not accepted", per AddParams.visibility's doc).
+  const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000'
+
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy(createTestConfig())
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  it('canonical + includeHidden carries an internal row AND the system row; round-trips through import', async () => {
+    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
+    const internalId = await brain.add({
+      data: 'Internal',
+      type: NounType.Thing,
+      subtype: 'x',
+      visibility: 'internal'
+    })
+
+    const migrationExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true })
+    const ids = migrationExport.entities.map((e) => e.id)
+    expect(ids).toContain(publicId)
+    expect(ids).toContain(internalId)
+    expect(ids).toContain(VFS_ROOT_ID)
+    expect(migrationExport.entities.find((e) => e.id === internalId)?.visibility).toBe('internal')
+    expect(migrationExport.entities.find((e) => e.id === VFS_ROOT_ID)?.visibility).toBe('system')
+
+    const target = new Brainy(createTestConfig())
+    await target.init()
+    try {
+      const result = await target.import(migrationExport)
+      expect(result.errors).toHaveLength(0)
+      expect((await target.get(internalId))?.visibility).toBe('internal')
+    } finally {
+      await target.close()
+    }
+  })
+
+  it('default export (includeHidden omitted) still excludes both hidden tiers — pins today\'s behavior', async () => {
+    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
+    const internalId = await brain.add({
+      data: 'Internal',
+      type: NounType.Thing,
+      subtype: 'x',
+      visibility: 'internal'
+    })
+
+    for (const opts of [{ enumeration: 'index' as const }, { enumeration: 'canonical' as const }]) {
+      const backup = await brain.export({}, opts)
+      const ids = backup.entities.map((e) => e.id)
+      expect(ids).toContain(publicId)
+      expect(ids).not.toContain(internalId)
+      expect(ids).not.toContain(VFS_ROOT_ID)
+    }
+  })
+
+  it('index mode + includeHidden also reaches both tiers — find() takes includeInternal + includeSystem in one pass', async () => {
+    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
+    const internalId = await brain.add({
+      data: 'Internal',
+      type: NounType.Thing,
+      subtype: 'x',
+      visibility: 'internal'
+    })
+
+    const indexExport = await brain.export({}, { enumeration: 'index', includeHidden: true })
+    const canonicalExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true })
+
+    const indexIds = indexExport.entities.map((e) => e.id).sort()
+    const canonicalIds = canonicalExport.entities.map((e) => e.id).sort()
+    expect(indexIds).toEqual(canonicalIds)
+    expect(indexIds).toContain(publicId)
+    expect(indexIds).toContain(internalId)
+    expect(indexIds).toContain(VFS_ROOT_ID)
+  })
+
+  it('drift stays pure under includeHidden — no tier-policy noise when the index is healthy', async () => {
+    await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
+    await brain.add({ data: 'Internal', type: NounType.Thing, subtype: 'x', visibility: 'internal' })
+
+    const audited = await brain.export(
+      {},
+      { enumeration: 'canonical', includeHidden: true, reportIndexDrift: true }
+    )
+    expect(audited.drift).toEqual({ canonicalOnly: [], indexOnly: [] })
+  })
+})