open-brainy/tests/unit/db/db-portable-graph.test.ts

/**
 * Unit tests for the 8.0 portable graph backup/restore surface:
 *   db.export() / brain.export()  (read, at a pinned generation)
 *   brain.import(backup)          (write, one atomic transaction; polymorphic)
 *
 * Real round-trips against in-memory storage (no mocks of the API under test),
 * including the 8.0-specific compositions (asOf/with export) and the polymorphic
 * import() dispatch. Entities carry subtypes so the suite passes under 8.0's
 * subtype-required default.
 */

import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { randomUUID } from 'node:crypto'
import * as fs from 'node:fs/promises'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../../src/brainy'
import { createTestConfig } from '../../helpers/test-factory'
import { NounType, VerbType } from '../../../src/types/graphTypes'
import { validatePortableGraph } from '../../../src/db/portableGraph'
import type { PortableGraph } from '../../../src/db/portableGraph'
import { CanonicalEnumerationUnavailableError } from '../../../src/db/errors'

describe('8.0 portable graph export/import (PortableGraph v1)', () => {
  let brain: Brainy

  beforeEach(async () => {
    brain = new Brainy(createTestConfig())
    await brain.init()
  })

  afterEach(async () => {
    await brain.close()
  })

  describe('format + round-trip', () => {
    it('brain.export() produces a versioned PortableGraph document', async () => {
      const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' })
      const b = await brain.add({ data: 'Acme', type: NounType.Organization, subtype: 'vendor' })
      await brain.relate({ from: a, to: b, type: VerbType.WorksWith, subtype: 'full-time' })

      const backup = await brain.export()
      expect(backup.format).toBe('brainy-portable-graph')
      expect(backup.formatVersion).toBe(1)
      expect(backup.entities.map((e) => e.id).sort()).toEqual([a, b].sort())
      expect(backup.relations).toHaveLength(1)
      expect(backup.entities.find((e) => e.id === a)?.subtype).toBe('employee')
      expect(backup.relations[0].subtype).toBe('full-time')
    })

    it('round-trips into a second brain via polymorphic brain.import()', async () => {
      const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' })
      const b = await brain.add({ data: 'Bob', type: NounType.Person, subtype: 'employee' })
      await brain.relate({ from: a, to: b, type: VerbType.FriendOf, subtype: 'close' })
      const backup = await brain.export({}, { includeVectors: true })

      const target = new Brainy(createTestConfig())
      await target.init()
      try {
        const result = await target.import(backup)
        expect(result.imported).toBe(2)
        expect(result.errors).toHaveLength(0)
        expect((await target.get(a))?.id).toBe(a)
        const rels = await target.related({ from: a })
        expect(rels.some((r) => r.to === b && r.type === VerbType.FriendOf)).toBe(true)
      } finally {
        await target.close()
      }
    })
  })

  describe('selectors', () => {
    it('ids — exactly the requested entities', async () => {
      const a = await brain.add({ data: 'A', type: NounType.Thing, subtype: 'x' })
      const b = await brain.add({ data: 'B', type: NounType.Thing, subtype: 'x' })
      await brain.add({ data: 'C', type: NounType.Thing, subtype: 'x' })
      const backup = await brain.export({ ids: [a, b] })
      expect(backup.entities.map((e) => e.id).sort()).toEqual([a, b].sort())
    })

    it('collection — collection + transitive Contains members', async () => {
      const root = await brain.add({ data: 'Folder', type: NounType.Collection, subtype: 'dir' })
      const c1 = await brain.add({ data: 'C1', type: NounType.Document, subtype: 'doc' })
      const c2 = await brain.add({ data: 'C2', type: NounType.Document, subtype: 'doc' })
      await brain.add({ data: 'Outside', type: NounType.Document, subtype: 'doc' })
      await brain.relate({ from: root, to: c1, type: VerbType.Contains, subtype: 'has' })
      await brain.relate({ from: c1, to: c2, type: VerbType.Contains, subtype: 'has' })
      const backup = await brain.export({ collection: root })
      expect(backup.entities.map((e) => e.id).sort()).toEqual([root, c1, c2].sort())
    })

    it('induced edges only by default', async () => {
      const a = await brain.add({ data: 'A', type: NounType.Thing, subtype: 'x' })
      const b = await brain.add({ data: 'B', type: NounType.Thing, subtype: 'x' })
      const c = await brain.add({ data: 'C', type: NounType.Thing, subtype: 'x' })
      await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'r' })
      await brain.relate({ from: b, to: c, type: VerbType.RelatedTo, subtype: 'r' })
      const backup = await brain.export({ ids: [a, b] })
      expect(backup.relations).toHaveLength(1)
      expect(backup.relations[0].from).toBe(a)
    })
  })

  describe('vectors + conflict', () => {
    it('omits vectors by default and re-embeds on import', async () => {
      const a = await brain.add({ data: 'Re-embed me', type: NounType.Thing, subtype: 'x' })
      const backup = await brain.export({ ids: [a] })
      expect(backup.entities[0].vector).toBeUndefined()

      const target = new Brainy(createTestConfig())
      await target.init()
      try {
        const result = await target.import(backup)
        expect(result.reembedded).toBe(1)
        expect((await target.get(a, { includeVectors: true }))?.vector?.length).toBeGreaterThan(0)
      } finally {
        await target.close()
      }
    })

    it('onConflict:merge updates an existing entity in place', async () => {
      const a = await brain.add({ data: 'V1', type: NounType.Thing, subtype: 'x', metadata: { n: 1 } })
      const backup = await brain.export({ ids: [a] }, { includeVectors: true })
      await brain.update({ id: a, metadata: { n: 99 }, merge: false })

      const result = await brain.import(backup, { onConflict: 'merge' })
      expect(result.merged).toBe(1)
      expect((await brain.get(a))?.metadata?.n).toBe(1)
    })
  })

  describe('8.0 composition — export on the immutable Db', () => {
    it('asOf(generation).export() is a time-travel export', async () => {
      // Only transact() advances the committed generation (add() does not), so
      // distinct generations are created via transact for a true time-travel pin.
      const a = '22222222-2222-4222-8222-222222222222'
      const b = '33333333-3333-4333-8333-333333333333'
      await brain.transact([{ op: 'add', id: a, data: 'First', type: NounType.Thing, subtype: 'x' }])
      const g1 = brain.generation()
      await brain.transact([{ op: 'add', id: b, data: 'Second', type: NounType.Thing, subtype: 'x' }])

      const past = await brain.asOf(g1)
      try {
        // Whole-brain export at g1 enumerates via the generation-correct find() path.
        const backup = await past.export()
        const ids = backup.entities.map((e) => e.id)
        expect(ids).toContain(a)
        expect(ids).not.toContain(b) // b was committed in a later generation
      } finally {
        await past.release()
      }
    })

    it('now().with(ops).export() is a what-if export', async () => {
      const a = await brain.add({ data: 'Real', type: NounType.Thing, subtype: 'x' })
      const speculativeId = '11111111-1111-4111-8111-111111111111'
      const view = await brain.now().with([
        { op: 'add', id: speculativeId, data: 'Speculative', type: NounType.Thing, subtype: 'x' }
      ])
      try {
        const backup = await view.export({ ids: [a, speculativeId] })
        expect(backup.entities.map((e) => e.id).sort()).toEqual([a, speculativeId].sort())
      } finally {
        await view.release()
      }
      // The speculative entity never touched the durable brain.
      expect(await brain.get(speculativeId)).toBeNull()
    })
  })

  describe('import validation', () => {
    it('rejects a newer formatVersion', async () => {
      const future: PortableGraph = {
        format: 'brainy-portable-graph',
        formatVersion: 999,
        brainyVersion: 'x',
        createdAt: new Date().toISOString(),
        embedding: { model: 'm', dimensions: 384 },
        entities: [],
        relations: [],
        stats: { entityCount: 0, relationCount: 0, blobCount: 0 }
      }
      await expect(brain.import(future)).rejects.toThrow(/formatVersion/)
    })
  })

  describe('id remapping (clone)', () => {
    it('imports a subgraph under fresh ids (a copy, not a move)', async () => {
      const a = await brain.add({ data: 'A', type: NounType.Thing, subtype: 'x' })
      const b = await brain.add({ data: 'B', type: NounType.Thing, subtype: 'x' })
      await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'r' })
      const backup = await brain.export({ ids: [a, b] }, { includeVectors: true })

      const remap = new Map<string, string>([
        [a, randomUUID()],
        [b, randomUUID()]
      ])
      const result = await brain.import(backup, { remapIds: (id) => remap.get(id) ?? id })
      expect(result.imported).toBe(2)
      expect((await brain.get(remap.get(a)!))?.id).toBe(remap.get(a))
      expect((await brain.get(a))?.id).toBe(a) // original still present
      const rels = await brain.related({ from: remap.get(a)! })
      expect(rels.some((r) => r.to === remap.get(b))).toBe(true)
    })
  })

  describe('validatePortableGraph() — dry-run check', () => {
    const valid = (): PortableGraph => ({
      format: 'brainy-portable-graph',
      formatVersion: 1,
      brainyVersion: 'x',
      createdAt: new Date().toISOString(),
      embedding: { model: 'all-MiniLM-L6-v2', dimensions: 384 },
      entities: [{ id: 'a', type: NounType.Thing }],
      relations: [],
      stats: { entityCount: 1, relationCount: 0, blobCount: 0, vectorDimensions: 384 }
    })

    it('accepts a well-formed backup', () => {
      const r = validatePortableGraph(valid())
      expect(r.valid).toBe(true)
      expect(r.errors).toHaveLength(0)
    })

    it('rejects a non-PortableGraph value', () => {
      const r = validatePortableGraph({ entities: [] })
      expect(r.valid).toBe(false)
      expect(r.errors[0]).toMatch(/PortableGraph/)
    })

    it('rejects a newer formatVersion', () => {
      const r = validatePortableGraph({ ...valid(), formatVersion: 999 })
      expect(r.valid).toBe(false)
      expect(r.errors.some((e) => /formatVersion/.test(e))).toBe(true)
    })

    it('flags duplicate entity ids as an error', () => {
      const b = valid()
      b.entities = [
        { id: 'dup', type: NounType.Thing },
        { id: 'dup', type: NounType.Thing }
      ]
      const r = validatePortableGraph(b)
      expect(r.valid).toBe(false)
      expect(r.errors.some((e) => /duplicate/.test(e))).toBe(true)
    })

    it('warns (not errors) on a relation endpoint missing from entities', () => {
      const b = valid()
      b.relations = [{ id: 'r1', from: 'a', to: 'missing', type: VerbType.RelatedTo }]
      const r = validatePortableGraph(b)
      expect(r.valid).toBe(true) // a dangling endpoint is a warning, not a hard error
      expect(r.warnings.some((w) => /missing/.test(w))).toBe(true)
    })
  })
})

describe('8.0 export includeContent (VFS blobs, filesystem)', () => {
  let dir: string
  let brain: Brainy

  beforeEach(async () => {
    dir = await fs.mkdtemp(path.join(os.tmpdir(), 'brainy-blob-'))
    brain = new Brainy({ storage: { type: 'filesystem', path: dir } })
    await brain.init()
  })

  afterEach(async () => {
    await brain.close()
    await fs.rm(dir, { recursive: true, force: true })
  })

  it('captures VFS file bytes in blobs and writes them on import', async () => {
    await brain.vfs.writeFile('/hello.txt', 'Hello blobs')

    const backup = await brain.export(
      { vfsPath: '/hello.txt' },
      { includeVectors: true, includeContent: true }
    )
    expect(backup.stats.blobCount).toBeGreaterThanOrEqual(1)
    const b64 = Object.values(backup.blobs ?? {})[0]
    expect(Buffer.from(b64, 'base64').toString()).toBe('Hello blobs')

    const dir2 = await fs.mkdtemp(path.join(os.tmpdir(), 'brainy-blob2-'))
    const target = new Brainy({ storage: { type: 'filesystem', path: dir2 } })
    await target.init()
    try {
      const result = await target.import(backup)
      expect(result.blobsWritten).toBeGreaterThanOrEqual(1)
      expect(result.imported).toBeGreaterThanOrEqual(1)
    } finally {
      await target.close()
      await fs.rm(dir2, { recursive: true, force: true })
    }
  })
})

describe('8.0 export enumeration:"canonical" — canon-complete against index blindness', () => {
  let brain: Brainy

  beforeEach(async () => {
    brain = new Brainy(createTestConfig())
    await brain.init()
  })

  afterEach(async () => {
    await brain.close()
  })

  it('(i) equals the index-based export when the index is healthy — same entity ids, relations, vectors', async () => {
    const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' })
    const b = await brain.add({ data: 'Bob', type: NounType.Person, subtype: 'employee' })
    const c = await brain.add({ data: 'Acme', type: NounType.Organization, subtype: 'vendor' })
    await brain.relate({ from: a, to: b, type: VerbType.FriendOf, subtype: 'close' })
    await brain.relate({ from: a, to: c, type: VerbType.WorksWith, subtype: 'full-time' })

    const indexExport = await brain.export({}, { includeVectors: true, enumeration: 'index' })
    const canonicalExport = await brain.export({}, { includeVectors: true, enumeration: 'canonical' })

    expect(canonicalExport.entities.map((e) => e.id).sort()).toEqual(
      indexExport.entities.map((e) => e.id).sort()
    )
    expect(canonicalExport.relations.map((r) => r.id).sort()).toEqual(
      indexExport.relations.map((r) => r.id).sort()
    )
    expect(canonicalExport.entities.map((e) => e.id).sort()).toEqual([a, b, c].sort())
    for (const e of canonicalExport.entities) {
      expect(e.vector?.length).toBeGreaterThan(0)
    }
    expect(canonicalExport.drift).toBeUndefined() // reportIndexDrift not requested
  })

  it('(ii) survives simulated metadata-index blindness; the index export misses the record; drift names it canonicalOnly', async () => {
    const staff = await brain.add({
      data: 'Staff',
      type: NounType.Person,
      subtype: 'employee',
      metadata: { role: 'staff' }
    })
    const other = await brain.add({
      data: 'Other',
      type: NounType.Person,
      subtype: 'employee',
      metadata: { role: 'staff' }
    })

    // Surgically poison the metadata index (the lowest-level seam the existing
    // find() phantom-row guard tests use — see find-index-integrity-guard.test.ts,
    // which does the mirror-image ADD case) so the predicate query
    // enumeration:'index' issues (find({ type: Person })) never returns `staff` —
    // a real canonical record the index has lost track of, the exact
    // canon-present/index-missing state canonical mode exists to survive.
    const mi = (brain as any).metadataIndex
    const original = mi.getIdsForFilter.bind(mi)
    mi.getIdsForFilter = async (filter: any, opts?: any): Promise<string[]> => {
      const ids: string[] = await original(filter, opts)
      return ids.filter((id: string) => id !== staff)
    }

    try {
      const indexExport = await brain.export({ type: NounType.Person }, { enumeration: 'index' })
      expect(indexExport.entities.map((e) => e.id)).not.toContain(staff)
      expect(indexExport.entities.map((e) => e.id)).toContain(other)

      const canonicalExport = await brain.export(
        { type: NounType.Person },
        { enumeration: 'canonical', reportIndexDrift: true }
      )
      expect(canonicalExport.entities.map((e) => e.id)).toContain(staff)
      expect(canonicalExport.entities.map((e) => e.id)).toContain(other)
      expect(canonicalExport.drift?.canonicalOnly).toEqual([staff])
      expect(canonicalExport.drift?.indexOnly).toEqual([])
    } finally {
      mi.getIdsForFilter = original
    }
  })

  it('(iii) drift report shape + loud console.warn only when nonzero', async () => {
    const staff = await brain.add({ data: 'Staff', type: NounType.Person, subtype: 'employee' })
    await brain.add({ data: 'Other', type: NounType.Person, subtype: 'employee' })

    const mi = (brain as any).metadataIndex
    const original = mi.getIdsForFilter.bind(mi)
    mi.getIdsForFilter = async (filter: any, opts?: any): Promise<string[]> => {
      const ids: string[] = await original(filter, opts)
      return ids.filter((id: string) => id !== staff)
    }

    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
    try {
      const drifted = await brain.export(
        { type: NounType.Person },
        { enumeration: 'canonical', reportIndexDrift: true }
      )
      expect(drifted.drift).toEqual({ canonicalOnly: [staff], indexOnly: [] })
      expect(warnSpy).toHaveBeenCalledTimes(1)
      expect(warnSpy.mock.calls[0].join(' ')).toMatch(/drift/i)
    } finally {
      mi.getIdsForFilter = original
      warnSpy.mockClear()
    }

    // Healthy index: drift is reported (both lists present) but never warned about.
    try {
      const healthy = await brain.export(
        { type: NounType.Person },
        { enumeration: 'canonical', reportIndexDrift: true }
      )
      expect(healthy.drift).toEqual({ canonicalOnly: [], indexOnly: [] })
      expect(warnSpy).not.toHaveBeenCalled()
    } finally {
      warnSpy.mockRestore()
    }
  })

  it('(iv) throws CanonicalEnumerationUnavailableError on a historical asOf() view and a speculative with() overlay', async () => {
    const a = '22222222-2222-4222-8222-222222222222'
    const b = '33333333-3333-4333-8333-333333333333'
    await brain.transact([{ op: 'add', id: a, data: 'First', type: NounType.Thing, subtype: 'x' }])
    const g1 = brain.generation()
    await brain.transact([{ op: 'add', id: b, data: 'Second', type: NounType.Thing, subtype: 'x' }])

    const past = await brain.asOf(g1)
    try {
      await expect(past.export({}, { enumeration: 'canonical' })).rejects.toThrow(
        CanonicalEnumerationUnavailableError
      )
      // The default (index) mode is unaffected — still a valid time-travel export.
      const backup = await past.export()
      expect(backup.entities.map((e) => e.id)).toContain(a)
    } finally {
      await past.release()
    }

    const speculativeId = '11111111-1111-4111-8111-111111111111'
    const view = await brain.now().with([
      { op: 'add', id: speculativeId, data: 'Speculative', type: NounType.Thing, subtype: 'x' }
    ])
    try {
      await expect(view.export({}, { enumeration: 'canonical' })).rejects.toThrow(
        CanonicalEnumerationUnavailableError
      )
    } finally {
      await view.release()
    }
  })

  it('throws a plain Error when enumeration:"canonical" has no storage adapter to walk', async () => {
    const { exportGraph } = await import('../../../src/db/portableGraph')
    const readerOnly = { get: async () => null, find: async () => [], related: async () => [] }
    await expect(
      exportGraph(readerOnly as any, undefined, {}, { enumeration: 'canonical' })
    ).rejects.toThrow(/enumeration:'canonical' requires a storage adapter/)
  })
})

describe('8.0 export includeHidden — every visibility tier for migration-grade canon completeness', () => {
  // The fixed-id VFS root Brainy.init() always creates is the one 'system'-visibility
  // entity a consumer can rely on existing (visibility:'system' is not settable via the
  // public add() API — "intentionally not accepted", per AddParams.visibility's doc).
  const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000'

  let brain: Brainy

  beforeEach(async () => {
    brain = new Brainy(createTestConfig())
    await brain.init()
  })

  afterEach(async () => {
    await brain.close()
  })

  it('canonical + includeHidden carries an internal row AND the system row; round-trips through import', async () => {
    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
    const internalId = await brain.add({
      data: 'Internal',
      type: NounType.Thing,
      subtype: 'x',
      visibility: 'internal'
    })

    const migrationExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true })
    const ids = migrationExport.entities.map((e) => e.id)
    expect(ids).toContain(publicId)
    expect(ids).toContain(internalId)
    expect(ids).toContain(VFS_ROOT_ID)
    expect(migrationExport.entities.find((e) => e.id === internalId)?.visibility).toBe('internal')
    expect(migrationExport.entities.find((e) => e.id === VFS_ROOT_ID)?.visibility).toBe('system')

    const target = new Brainy(createTestConfig())
    await target.init()
    try {
      const result = await target.import(migrationExport)
      expect(result.errors).toHaveLength(0)
      expect((await target.get(internalId))?.visibility).toBe('internal')
    } finally {
      await target.close()
    }
  })

  it('default export (includeHidden omitted) still excludes both hidden tiers — pins today\'s behavior', async () => {
    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
    const internalId = await brain.add({
      data: 'Internal',
      type: NounType.Thing,
      subtype: 'x',
      visibility: 'internal'
    })

    for (const opts of [{ enumeration: 'index' as const }, { enumeration: 'canonical' as const }]) {
      const backup = await brain.export({}, opts)
      const ids = backup.entities.map((e) => e.id)
      expect(ids).toContain(publicId)
      expect(ids).not.toContain(internalId)
      expect(ids).not.toContain(VFS_ROOT_ID)
    }
  })

  it('index mode + includeHidden also reaches both tiers — find() takes includeInternal + includeSystem in one pass', async () => {
    const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
    const internalId = await brain.add({
      data: 'Internal',
      type: NounType.Thing,
      subtype: 'x',
      visibility: 'internal'
    })

    const indexExport = await brain.export({}, { enumeration: 'index', includeHidden: true })
    const canonicalExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true })

    const indexIds = indexExport.entities.map((e) => e.id).sort()
    const canonicalIds = canonicalExport.entities.map((e) => e.id).sort()
    expect(indexIds).toEqual(canonicalIds)
    expect(indexIds).toContain(publicId)
    expect(indexIds).toContain(internalId)
    expect(indexIds).toContain(VFS_ROOT_ID)
  })

  it('drift stays pure under includeHidden — no tier-policy noise when the index is healthy', async () => {
    await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
    await brain.add({ data: 'Internal', type: NounType.Thing, subtype: 'x', visibility: 'internal' })

    const audited = await brain.export(
      {},
      { enumeration: 'canonical', includeHidden: true, reportIndexDrift: true }
    )
    expect(audited.drift).toEqual({ canonicalOnly: [], indexOnly: [] })
  })
})