David Snelling
a93bb4e85f
The one-time 7→8 layout migration rescues the head branch's entities (branches/<head>/entities/* → entities/*), then drained the entire branches/<head>/ directory. Any durable state a 7.x engine wrote under the head branch OUTSIDE entities/ — a branch-scoped index, blob area, or field registry — would be silently deleted by that drain, the same failure class as the VFS content blobs a 7.x store kept in _cow/. Guard it: after the entity move, list what remains under branches/<head>/ and exclude entities/. If anything survives, it is non-entity durable state, so PRESERVE the branch (skip removeRawPrefix) and warn loudly with the leftover keys — recoverable, not lost. A clean branch (only the moved entities) still drains exactly as before. Adds a PARITY GUARD test to the 7→8 migration suite.
229 lines
8.9 KiB
TypeScript
229 lines
8.9 KiB
TypeScript
/**
|
|
* @module tests/integration/migration-7x-to-8x
|
|
* @description Proof suite for the 7.x → 8.0 on-disk layout migration. 7.x stored
|
|
* every object branch-scoped under `branches/<branch>/<basePath>`; 8.0 stores the
|
|
* identical entity structure at the ROOT (`entities/<kind>/<shard>/<id>/…`) plus
|
|
* the generational `_system/manifest.json` + `_generations/`. GenerationStore.open
|
|
* is TOLERANT of a missing manifest (opens at generation 0), so a naive 8.0 open of
|
|
* a 7.x directory reports zero entities and SILENTLY LOSES ALL DATA. The migration
|
|
* collapses the HEAD branch's entities to the root before storage is set up.
|
|
*
|
|
* Tests:
|
|
* 1. DATA-LOSS LOCK — a built-in FileSystemStorage opened on the reshaped 7.x dir
|
|
* reports zero nouns at the root layout (the mode the migration fixes; durable
|
|
* because storage never migrates).
|
|
* 2. ROUND TRIP — opening a brain with autoMigrate (default) restores
|
|
* find/related/counts byte-equal to the reference captured before reshaping.
|
|
* 3. IDEMPOTENCY — a second open is a no-op (marker short-circuits; branch drained).
|
|
* 4. GUARD — autoMigrate:false on a legacy layout throws explicit guidance.
|
|
* 5. NO-OP — a native flat 8.0 brain opens untouched (no marker churn, no data move).
|
|
*/
|
|
import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'
|
|
import * as fs from 'node:fs'
|
|
import * as os from 'node:os'
|
|
import * as path from 'node:path'
|
|
import { Brainy } from '../../src/brainy.js'
|
|
import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
|
|
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
|
|
|
interface Reference {
|
|
nounCount: number
|
|
verbCount: number
|
|
docIds: string[]
|
|
activeIds: string[]
|
|
relatedFrom0: string[]
|
|
}
|
|
|
|
const docId = (i: number): string => `00000000-0000-4000-8000-00000000000${i}`
|
|
|
|
async function buildReferenceBrain(dir: string): Promise<Reference> {
|
|
const brain = new Brainy({
|
|
requireSubtype: false,
|
|
storage: { type: 'filesystem', path: dir },
|
|
dimensions: 384,
|
|
silent: true
|
|
})
|
|
await brain.init()
|
|
for (let i = 0; i < 5; i++) {
|
|
await brain.add({
|
|
id: docId(i),
|
|
data: `Document number ${i} about distributed systems`,
|
|
type: NounType.Document,
|
|
metadata: { title: `Doc ${i}`, status: i % 2 === 0 ? 'active' : 'archived' }
|
|
})
|
|
}
|
|
await brain.relate({ from: docId(0), to: docId(1), type: VerbType.RelatedTo })
|
|
await brain.relate({ from: docId(0), to: docId(2), type: VerbType.RelatedTo })
|
|
await brain.flush()
|
|
const ref = await captureReference(brain)
|
|
await brain.close()
|
|
return ref
|
|
}
|
|
|
|
async function captureReference(brain: Brainy): Promise<Reference> {
|
|
const docs = await brain.find({ type: NounType.Document, limit: 100 })
|
|
const active = await brain.find({ type: NounType.Document, where: { status: 'active' }, limit: 100 })
|
|
const related = await brain.related(docId(0))
|
|
return {
|
|
nounCount: await brain.getNounCount(),
|
|
verbCount: await brain.getVerbCount(),
|
|
docIds: docs.map((r: any) => r.id).sort(),
|
|
activeIds: active.map((r: any) => r.id).sort(),
|
|
relatedFrom0: related.map((r: any) => r.to).sort()
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Reshape a flat 8.0 directory into the 7.x branch layout: move `entities/` under
|
|
* `branches/main/entities/`, and delete the 8.0-only derived + MVCC state (8.0
|
|
* rebuilds all of it from the canonical entities). The result is what a 7.x store
|
|
* looks like to 8.0: canonical entities present, but invisible at the root.
|
|
*/
|
|
function reshapeToLegacy(dir: string): void {
|
|
const branchEntities = path.join(dir, 'branches', 'main', 'entities')
|
|
fs.mkdirSync(path.dirname(branchEntities), { recursive: true })
|
|
fs.renameSync(path.join(dir, 'entities'), branchEntities)
|
|
for (const derived of ['_system', '_column_index', '_blobs', '_generations', 'indexes']) {
|
|
fs.rmSync(path.join(dir, derived), { recursive: true, force: true })
|
|
}
|
|
}
|
|
|
|
const makeTempDir = (): string => fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-migrate-'))
|
|
|
|
async function openBrain(dir: string, extra: Record<string, unknown> = {}): Promise<Brainy> {
|
|
return new Brainy({
|
|
requireSubtype: false,
|
|
storage: { type: 'filesystem', path: dir },
|
|
dimensions: 384,
|
|
silent: true,
|
|
...extra
|
|
})
|
|
}
|
|
|
|
describe('7.x → 8.0 layout migration', () => {
|
|
const dirs: string[] = []
|
|
const brains: Brainy[] = []
|
|
let reference: Reference
|
|
let legacyTemplate: string
|
|
|
|
beforeAll(async () => {
|
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
|
legacyTemplate = makeTempDir()
|
|
reference = await buildReferenceBrain(legacyTemplate)
|
|
reshapeToLegacy(legacyTemplate)
|
|
})
|
|
|
|
afterAll(() => fs.rmSync(legacyTemplate, { recursive: true, force: true }))
|
|
|
|
afterEach(async () => {
|
|
for (const b of brains.splice(0)) {
|
|
try {
|
|
await b.close()
|
|
} catch {
|
|
/* already closed */
|
|
}
|
|
}
|
|
for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
|
|
})
|
|
|
|
/** A fresh copy of the reshaped legacy directory for one test to mutate. */
|
|
function freshLegacyDir(): string {
|
|
const dir = makeTempDir()
|
|
dirs.push(dir)
|
|
fs.cpSync(legacyTemplate, dir, { recursive: true })
|
|
return dir
|
|
}
|
|
|
|
it('DATA-LOSS LOCK: a built-in FileSystemStorage on a 7.x layout reports zero nouns at the root', async () => {
|
|
const dir = freshLegacyDir()
|
|
const storage = new FileSystemStorage(dir)
|
|
await storage.init()
|
|
// The canonical entities exist on disk (under branches/main/) but are invisible
|
|
// at the 8.0 root layout — exactly the silent data loss the migration prevents.
|
|
expect(await storage.getNounCount()).toBe(0)
|
|
})
|
|
|
|
it('ROUND TRIP: opening with autoMigrate (default) restores all three intelligences byte-equal', async () => {
|
|
const dir = freshLegacyDir()
|
|
const brain = await openBrain(dir)
|
|
brains.push(brain)
|
|
await brain.init()
|
|
const after = await captureReference(brain)
|
|
expect(after).toEqual(reference)
|
|
// The branch layout is drained and the marker stamped.
|
|
expect(fs.existsSync(path.join(dir, 'branches', 'main', 'entities'))).toBe(false)
|
|
expect(fs.existsSync(path.join(dir, 'entities'))).toBe(true)
|
|
})
|
|
|
|
it('IDEMPOTENCY: a second open is a no-op and still reads the migrated data', async () => {
|
|
const dir = freshLegacyDir()
|
|
const first = await openBrain(dir)
|
|
brains.push(first)
|
|
await first.init()
|
|
await first.close()
|
|
brains.splice(brains.indexOf(first), 1)
|
|
|
|
// Mtime of the marker must not change on the second open (no re-migration).
|
|
const markerPath = path.join(dir, '_system', 'migration-layout.json')
|
|
const markerExists = fs.existsSync(markerPath) || fs.existsSync(markerPath + '.gz')
|
|
|
|
const second = await openBrain(dir)
|
|
brains.push(second)
|
|
await second.init()
|
|
expect(markerExists).toBe(true)
|
|
expect(await captureReference(second)).toEqual(reference)
|
|
})
|
|
|
|
it('GUARD: autoMigrate:false on a 7.x layout throws explicit guidance instead of losing data', async () => {
|
|
const dir = freshLegacyDir()
|
|
const brain = await openBrain(dir, { autoMigrate: false })
|
|
brains.push(brain)
|
|
await expect(brain.init()).rejects.toThrow(/7\.x branch layout/)
|
|
})
|
|
|
|
it('NO-OP: a native flat 8.0 brain opens untouched', async () => {
|
|
const dir = makeTempDir()
|
|
dirs.push(dir)
|
|
const ref = await buildReferenceBrain(dir) // builds + closes a normal flat brain
|
|
const reopened = await openBrain(dir)
|
|
brains.push(reopened)
|
|
await reopened.init() // migration phase is a strict no-op here
|
|
expect(await captureReference(reopened)).toEqual(ref)
|
|
expect(fs.existsSync(path.join(dir, 'branches'))).toBe(false)
|
|
})
|
|
|
|
it('PARITY GUARD: branch-scoped non-entity state survives migration (not silently drained)', async () => {
|
|
const dir = freshLegacyDir()
|
|
|
|
// Simulate a 7.x engine that wrote durable state under the HEAD branch
|
|
// OUTSIDE entities/ (a branch-scoped index/blob/registry). The migration
|
|
// rescues entities/ only; the guard must PRESERVE the rest rather than let
|
|
// removeRawPrefix silently delete it — the VFS `_cow/` stranding lesson.
|
|
const raw: any = new FileSystemStorage(dir)
|
|
await raw.init()
|
|
await raw.writeRawObject('branches/main/_legacy_engine/state', { keep: 'me', n: 42 })
|
|
await raw.flush?.()
|
|
raw.stopFlushRequestWatcher?.()
|
|
await raw.releaseWriterLock?.()
|
|
|
|
// Migrate (entities collapse to the root as usual).
|
|
const brain = await openBrain(dir)
|
|
brains.push(brain)
|
|
await brain.init()
|
|
expect(await captureReference(brain)).toEqual(reference)
|
|
await brain.close()
|
|
brains.splice(brains.indexOf(brain), 1)
|
|
|
|
// The non-entity branch-scoped state is PRESERVED (branch NOT drained) —
|
|
// recoverable, not lost.
|
|
const check: any = new FileSystemStorage(dir)
|
|
await check.init()
|
|
expect(await check.readRawObject('branches/main/_legacy_engine/state')).toMatchObject({
|
|
keep: 'me',
|
|
n: 42
|
|
})
|
|
await check.flush?.()
|
|
check.stopFlushRequestWatcher?.()
|
|
await check.releaseWriterLock?.()
|
|
})
|
|
})
|