open-brainy/tests/integration/migration-7x-to-8x.test.ts

/**
 * @module tests/integration/migration-7x-to-8x
 * @description Proof suite for the 7.x → 8.0 on-disk layout migration. 7.x stored
 * every object branch-scoped under `branches/<branch>/<basePath>`; 8.0 stores the
 * identical entity structure at the ROOT (`entities/<kind>/<shard>/<id>/…`) plus
 * the generational `_system/manifest.json` + `_generations/`. GenerationStore.open
 * is TOLERANT of a missing manifest (opens at generation 0), so a naive 8.0 open of
 * a 7.x directory reports zero entities and SILENTLY LOSES ALL DATA. The migration
 * collapses the HEAD branch's entities to the root before storage is set up.
 *
 * Tests:
 *  1. DATA-LOSS LOCK — a built-in FileSystemStorage opened on the reshaped 7.x dir
 *     reports zero nouns at the root layout (the mode the migration fixes; durable
 *     because storage never migrates).
 *  2. ROUND TRIP — opening a brain with autoMigrate (default) restores
 *     find/related/counts byte-equal to the reference captured before reshaping.
 *  3. IDEMPOTENCY — a second open is a no-op (marker short-circuits; branch drained).
 *  4. GUARD — autoMigrate:false on a legacy layout throws explicit guidance.
 *  5. NO-OP — a native flat 8.0 brain opens untouched (no marker churn, no data move).
 */
import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/brainy.js'
import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
import { NounType, VerbType } from '../../src/types/graphTypes.js'

interface Reference {
  nounCount: number
  verbCount: number
  docIds: string[]
  activeIds: string[]
  relatedFrom0: string[]
}

const docId = (i: number): string => `00000000-0000-4000-8000-00000000000${i}`

async function buildReferenceBrain(dir: string): Promise<Reference> {
  const brain = new Brainy({
    requireSubtype: false,
    storage: { type: 'filesystem', path: dir },
    dimensions: 384,
    silent: true
  })
  await brain.init()
  for (let i = 0; i < 5; i++) {
    await brain.add({
      id: docId(i),
      data: `Document number ${i} about distributed systems`,
      type: NounType.Document,
      metadata: { title: `Doc ${i}`, status: i % 2 === 0 ? 'active' : 'archived' }
    })
  }
  await brain.relate({ from: docId(0), to: docId(1), type: VerbType.RelatedTo })
  await brain.relate({ from: docId(0), to: docId(2), type: VerbType.RelatedTo })
  await brain.flush()
  const ref = await captureReference(brain)
  await brain.close()
  return ref
}

async function captureReference(brain: Brainy): Promise<Reference> {
  const docs = await brain.find({ type: NounType.Document, limit: 100 })
  const active = await brain.find({ type: NounType.Document, where: { status: 'active' }, limit: 100 })
  const related = await brain.related(docId(0))
  return {
    nounCount: await brain.getNounCount(),
    verbCount: await brain.getVerbCount(),
    docIds: docs.map((r: any) => r.id).sort(),
    activeIds: active.map((r: any) => r.id).sort(),
    relatedFrom0: related.map((r: any) => r.to).sort()
  }
}

/**
 * Reshape a flat 8.0 directory into the 7.x branch layout: move `entities/` under
 * `branches/main/entities/`, and delete the 8.0-only derived + MVCC state (8.0
 * rebuilds all of it from the canonical entities). The result is what a 7.x store
 * looks like to 8.0: canonical entities present, but invisible at the root.
 */
function reshapeToLegacy(dir: string): void {
  const branchEntities = path.join(dir, 'branches', 'main', 'entities')
  fs.mkdirSync(path.dirname(branchEntities), { recursive: true })
  fs.renameSync(path.join(dir, 'entities'), branchEntities)
  for (const derived of ['_system', '_column_index', '_blobs', '_generations', 'indexes']) {
    fs.rmSync(path.join(dir, derived), { recursive: true, force: true })
  }
}

const makeTempDir = (): string => fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-migrate-'))

async function openBrain(dir: string, extra: Record<string, unknown> = {}): Promise<Brainy> {
  return new Brainy({
    requireSubtype: false,
    storage: { type: 'filesystem', path: dir },
    dimensions: 384,
    silent: true,
    ...extra
  })
}

describe('7.x → 8.0 layout migration', () => {
  const dirs: string[] = []
  const brains: Brainy[] = []
  let reference: Reference
  let legacyTemplate: string

  beforeAll(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    legacyTemplate = makeTempDir()
    reference = await buildReferenceBrain(legacyTemplate)
    reshapeToLegacy(legacyTemplate)
  })

  afterAll(() => fs.rmSync(legacyTemplate, { recursive: true, force: true }))

  afterEach(async () => {
    for (const b of brains.splice(0)) {
      try {
        await b.close()
      } catch {
        /* already closed */
      }
    }
    for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
  })

  /** A fresh copy of the reshaped legacy directory for one test to mutate. */
  function freshLegacyDir(): string {
    const dir = makeTempDir()
    dirs.push(dir)
    fs.cpSync(legacyTemplate, dir, { recursive: true })
    return dir
  }

  it('DATA-LOSS LOCK: a built-in FileSystemStorage on a 7.x layout reports zero nouns at the root', async () => {
    const dir = freshLegacyDir()
    const storage = new FileSystemStorage(dir)
    await storage.init()
    // The canonical entities exist on disk (under branches/main/) but are invisible
    // at the 8.0 root layout — exactly the silent data loss the migration prevents.
    expect(await storage.getNounCount()).toBe(0)
  })

  it('ROUND TRIP: opening with autoMigrate (default) restores all three intelligences byte-equal', async () => {
    const dir = freshLegacyDir()
    const brain = await openBrain(dir)
    brains.push(brain)
    await brain.init()
    const after = await captureReference(brain)
    expect(after).toEqual(reference)
    // The branch layout is drained and the marker stamped.
    expect(fs.existsSync(path.join(dir, 'branches', 'main', 'entities'))).toBe(false)
    expect(fs.existsSync(path.join(dir, 'entities'))).toBe(true)
  })

  it('IDEMPOTENCY: a second open is a no-op and still reads the migrated data', async () => {
    const dir = freshLegacyDir()
    const first = await openBrain(dir)
    brains.push(first)
    await first.init()
    await first.close()
    brains.splice(brains.indexOf(first), 1)

    // Mtime of the marker must not change on the second open (no re-migration).
    const markerPath = path.join(dir, '_system', 'migration-layout.json')
    const markerExists = fs.existsSync(markerPath) || fs.existsSync(markerPath + '.gz')

    const second = await openBrain(dir)
    brains.push(second)
    await second.init()
    expect(markerExists).toBe(true)
    expect(await captureReference(second)).toEqual(reference)
  })

  it('GUARD: autoMigrate:false on a 7.x layout throws explicit guidance instead of losing data', async () => {
    const dir = freshLegacyDir()
    const brain = await openBrain(dir, { autoMigrate: false })
    brains.push(brain)
    await expect(brain.init()).rejects.toThrow(/7\.x branch layout/)
  })

  it('NO-OP: a native flat 8.0 brain opens untouched', async () => {
    const dir = makeTempDir()
    dirs.push(dir)
    const ref = await buildReferenceBrain(dir) // builds + closes a normal flat brain
    const reopened = await openBrain(dir)
    brains.push(reopened)
    await reopened.init() // migration phase is a strict no-op here
    expect(await captureReference(reopened)).toEqual(ref)
    expect(fs.existsSync(path.join(dir, 'branches'))).toBe(false)
  })

  it('PARITY GUARD: branch-scoped non-entity state survives migration (not silently drained)', async () => {
    const dir = freshLegacyDir()

    // Simulate a 7.x engine that wrote durable state under the HEAD branch
    // OUTSIDE entities/ (a branch-scoped index/blob/registry). The migration
    // rescues entities/ only; the guard must PRESERVE the rest rather than let
    // removeRawPrefix silently delete it — the VFS `_cow/` stranding lesson.
    const raw: any = new FileSystemStorage(dir)
    await raw.init()
    await raw.writeRawObject('branches/main/_legacy_engine/state', { keep: 'me', n: 42 })
    await raw.flush?.()
    raw.stopFlushRequestWatcher?.()
    await raw.releaseWriterLock?.()

    // Migrate (entities collapse to the root as usual).
    const brain = await openBrain(dir)
    brains.push(brain)
    await brain.init()
    expect(await captureReference(brain)).toEqual(reference)
    await brain.close()
    brains.splice(brains.indexOf(brain), 1)

    // The non-entity branch-scoped state is PRESERVED (branch NOT drained) —
    // recoverable, not lost.
    const check: any = new FileSystemStorage(dir)
    await check.init()
    expect(await check.readRawObject('branches/main/_legacy_engine/state')).toMatchObject({
      keep: 'me',
      n: 42
    })
    await check.flush?.()
    check.stopFlushRequestWatcher?.()
    await check.releaseWriterLock?.()
  })
})