From 0c4a51c24e8e1466352a52a37fe86b7379fdb4d4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 19 Jun 2026 13:44:03 -0700 Subject: [PATCH] =?UTF-8?q?feat(8.0):=207.x=E2=86=928.0=20layout=20migrati?= =?UTF-8?q?on=20=E2=80=94=20fix=20silent=20total=20data=20loss=20on=20firs?= =?UTF-8?q?t=20open?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7.x stored every object branch-scoped under branches//; 8.0 stores the IDENTICAL entity structure at the storage ROOT plus the generational _system/_generations layer. GenerationStore.open tolerates a missing manifest (opens at gen 0), so a naive 8.0 open of a 7.x directory reported ZERO entities and SILENTLY LOST ALL DATA. The fix is blocked in the normal init order — cor 3.0's storage-factory legacy guard fires inside setupStorage(), five steps before the old migration check — so a new phase runs BEFORE setupStorage(). legacyLayoutMigrationPhase() (init, between loadPlugins and setupStorage): - Fast-path: returns immediately when there's no top-level branches/ dir (every native 8.0 brain + fresh dir pays only one fs.existsSync on the init hot path). - Runs on a temporary BUILT-IN FileSystemStorage (never the plugin adapter, whose guard would throw). Memory/cloud/pre-built-adapter stores are a strict no-op. - Detect via _system/migration-layout.json marker + branches//entities/. - autoMigrate:false on a legacy layout THROWS explicit guidance (no silent loss). - Acquire the writer lock, then COLLAPSE branches//entities/* → entities/* via the .gz-transparent raw primitives (read→write→delete, idempotent/resume-safe; NOT fs.rename — logical paths + memory-adapter incompatibility). - Rebuild persisted counts (rebuildCounts → totalNounCount/counts.json; rebuildTypeCounts/rebuildSubtypeCounts → per-type stats) — the three indexes are rebuilt by the normal init's rebuildIndexesIfNeeded afterward. - Finalize: stamp the flat-v8 marker (re-open no-op), drain the head branch (non-head branches = 7.x version history 8.0's MVCC does not import; left as-is). Tests (tests/integration/migration-7x-to-8x.test.ts): the data-loss LOCK (a built-in FileSystemStorage on a reshaped 7.x dir reports 0 nouns), a byte-equal ROUND TRIP (find/related/counts equal the pre-reshape reference), IDEMPOTENCY, the autoMigrate:false GUARD throw, and the native-flat NO-OP. RELEASES upgrade checklist rewritten (the old note documented the opposite, lossy behavior). 1477 unit + migration 5 + db-mvcc 25 green; no init-path regression. Follow-ups (hardening, not correctness for the common case): backupTo config plumbing (currently a loud warning), a crash-mid-collapse resume test, and a boundary-safe fake-plugin ordering test. --- RELEASES.md | 16 +- src/brainy.ts | 168 ++++++++++++++++ tests/integration/migration-7x-to-8x.test.ts | 194 +++++++++++++++++++ 3 files changed, 375 insertions(+), 3 deletions(-) create mode 100644 tests/integration/migration-7x-to-8x.test.ts diff --git a/RELEASES.md b/RELEASES.md index bbca3343..e95dc9a7 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -160,9 +160,19 @@ write-time `sum`/`count`/`avg`/`min`/`max`. | `BrainyZeroConfig` type | Removed. 8.0 zero-config is automatic and internal — `new Brainy()` auto-detects storage, derives HNSW quality from `config.vector.recall`, picks `persistMode` from the adapter, and sizes caches to the detected container-memory limit. There is no config-generation type to import. | | `brain.isFullyInitialized()` / `brain.awaitBackgroundInit()` | `await brain.ready`. The built-in filesystem/memory adapters finish initialization synchronously inside `init()`, so a single readiness promise is the whole story — these methods were no-ops once cloud adapters were removed. | -If you used 7.x COW branches, **materialize every branch you care about while -still on 7.x** (fork → export, or copy each branch's store) — 8.0 does not -read COW branch state. +**Opening a 7.x store auto-migrates it.** 8.0 stores entities at the root; 7.x +stored them branch-scoped under `branches//`. On first open, 8.0 collapses +the **HEAD branch** (`config.storage.branch`, default `main`) to the 8.0 layout in +place and rebuilds all derived state — no action required (`autoMigrate` defaults +to `true`; set it to `false` to make 8.0 refuse a legacy layout with an explicit +error instead). Two caveats: + +- **Non-HEAD branches are not imported.** 8.0 has no COW branches; only the head + branch's data is migrated. If you care about other branches, **export each one + while still on 7.x** (`fork → export`, or copy its store). +- **Back up first for rollback.** The migration mutates the directory in place and + 8.0 does not keep the old layout — copy the data directory before the first 8.0 + open if you need to roll back. ### Renames — find/replace pairs diff --git a/src/brainy.ts b/src/brainy.ts index d4b8e017..d71745f0 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -11,6 +11,7 @@ import { JsHnswVectorIndex } from './hnsw/hnswIndex.js' // for the 99% of queries that don't filter by type (avoids searching 42 separate graphs) import { createStorage } from './storage/storageFactory.js' import type { StorageOptions } from './storage/storageFactory.js' +import { rebuildCounts } from './utils/rebuildCounts.js' import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js' import { BaseStorage } from './storage/baseStorage.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' @@ -650,6 +651,12 @@ export class Brainy implements BrainyInterface { // so plugin-provided storage factories (e.g., filesystem override from cortex) are available await this.loadPlugins() + // 7.x → 8.0 layout migration, BEFORE storage setup: collapse a legacy + // branch layout to the flat 8.0 layout on a built-in FileSystemStorage, so + // setupStorage() (and any native plugin factory's own legacy guard) sees a + // clean store. No-op for non-filesystem stores and already-flat brains. + await this.legacyLayoutMigrationPhase() + // Setup and initialize storage (checks plugin storage factories first) this.storage = await this.setupStorage() await this.storage.init() @@ -10748,6 +10755,167 @@ export class Brainy implements BrainyInterface { /** * Setup storage */ + /** + * @description 7.x → 8.0 on-disk LAYOUT migration, run BEFORE `setupStorage()`. + * + * 7.x stored every object branch-scoped under `branches//`; + * 8.0 stores the IDENTICAL entity structure at the storage ROOT + * (`entities////…`) plus the generational `_system` + + * `_generations` layer. `GenerationStore.open()` is tolerant of a missing + * manifest (opens at generation 0), so a naive 8.0 open of a 7.x directory + * reports zero entities and SILENTLY LOSES ALL DATA. This phase collapses the + * HEAD branch's canonical entities to the root so the rest of init (and any + * native plugin storage factory, whose own legacy guard would otherwise THROW) + * sees a clean flat-v8 store; 8.0 rebuilds all derived state from those entities. + * + * Runs on a temporary BUILT-IN `FileSystemStorage` (never the plugin adapter), + * is idempotent (a `_system/migration-layout.json` marker short-circuits re-open), + * resume-safe (per-object read→write→delete), and a strict NO-OP for any brain + * that is not a filesystem store carrying a legacy `branches/` layout. + */ + private async legacyLayoutMigrationPhase(): Promise { + // Pre-built adapter instances and non-filesystem stores have no on-disk 7.x + // layout to migrate. + if (isStorageAdapterInstance(this.config.storage)) return + const storageConfig = (this.config.storage || {}) as StorageOptions & Record + const type = storageConfig.type || 'auto' + if (type !== 'filesystem' && type !== 'auto') return + + // Fast path: the only on-disk shape that needs migrating has a top-level + // `branches/` directory. A native 8.0 brain and a fresh directory do not, so + // skip the probe-storage construction entirely on the init hot path. (The + // migration is node-filesystem-only; `fs.existsSync` is absent/throwing + // elsewhere, which the catch treats as "nothing to migrate".) + const rootDir = + (storageConfig.rootDirectory as string) || + (storageConfig.path as string) || + './brainy-data' // createStorage's filesystem default + try { + if (!fs.existsSync(`${rootDir}/branches`)) return + } catch { + return + } + + // Probe with a BUILT-IN filesystem storage over the same root (never the + // plugin factory — cor's adapter would throw on the legacy layout). In a + // non-filesystem environment this throws; that just means "nothing to migrate". + let probe: BaseStorage + try { + probe = (await createStorage({ ...storageConfig, type: 'filesystem' })) as BaseStorage + await probe.init() + } catch { + return + } + + try { + // Idempotency: a completed migration leaves a marker → re-open is a no-op. + const marker = await probe.readRawObject('_system/migration-layout.json') + if (marker && (marker as { layout?: string }).layout === 'flat-v8') return + + // DETECT: HEAD-branch entities present under branches//entities/. + const head = (storageConfig.branch as string) || 'main' + const branchPaths = await probe.listRawObjects('branches') + const headEntityPrefix = `branches/${head}/entities/` + const legacyEntityPaths = branchPaths.filter((p) => p.startsWith(headEntityPrefix)) + + if (legacyEntityPaths.length === 0) { + // Already flat (root entities, no head-branch entities) → stamp the marker + // so future opens short-circuit. A genuinely empty/fresh dir gets no marker. + const rootEntities = await probe.listRawObjects('entities') + if (rootEntities.length > 0) { + await probe.writeRawObject('_system/migration-layout.json', { + layout: 'flat-v8', + version: 8, + fromBranch: null, + entitiesMoved: 0 + }) + } + return + } + + // GUARD: an explicit autoMigrate:false must not silently lose 7.x data. + if (this.config.autoMigrate === false) { + throw new Error( + `Brainy 8.0 found a 7.x branch layout at this storage directory ` + + `(${legacyEntityPaths.length} entities under branches/${head}/). Opening it as-is ` + + `would report zero entities and lose your data. Set autoMigrate: true (the default) ` + + `to migrate the layout in place on open, or export the data on 7.x first.` + ) + } + + // Acquire the writer lock for the duration of the move (idempotent marker + // makes a lost race harmless, but the lock prevents two concurrent collapses). + const canLock = + typeof (probe as unknown as { supportsMultiProcessLocking?: () => boolean }) + .supportsMultiProcessLocking === 'function' && + (probe as unknown as { supportsMultiProcessLocking: () => boolean }).supportsMultiProcessLocking() + const lockable = probe as unknown as { + acquireWriterLock?: (o: { force?: boolean }) => Promise + releaseWriterLock?: () => Promise + } + if (canLock && typeof lockable.acquireWriterLock === 'function') { + await lockable.acquireWriterLock({ force: this.config.force }) + } + + if (!this.config.silent) { + console.warn( + `[brainy] Migrating a 7.x branch layout (branches/${head}) to the 8.0 flat layout ` + + `in place — ${legacyEntityPaths.length} entity files. This runs once; back up the ` + + `directory first if you need a rollback (8.0 does not keep the old layout).` + ) + } + + try { + // COLLAPSE: branches//entities/* → entities/* via the .gz-transparent + // raw primitives (NOT fs.rename — listRawObjects returns logical paths and + // the in-memory adapter has no real files). read→write→delete is idempotent, + // so a crash mid-move is recovered by simply re-running. + let moved = 0 + for (const src of legacyEntityPaths) { + const target = src.slice(`branches/${head}/`.length) // → entities/... + const data = await probe.readRawObject(src) + if (data === null) continue // already moved by an interrupted prior run + await probe.writeRawObject(target, data) + await probe.deleteRawObject(src) + moved++ + } + + // REBUILD persisted derived state from the now-flat canonical entities. + // (The three indexes — HNSW, metadata, graph — are rebuilt by the normal + // init's rebuildIndexesIfNeeded after storage is set up; the COUNT rollups + // are NOT, so they must be rebuilt + persisted here or getNounCount()/stats() + // read 0 on the migrated store.) + // - rebuildCounts → totalNounCount/totalVerbCount + counts.json (getNounCount/getVerbCount) + // - rebuildTypeCounts → per-type type-statistics.json (stats().entitiesByType) + await rebuildCounts(probe) + await probe.rebuildTypeCounts() + if (typeof (probe as unknown as { rebuildSubtypeCounts?: () => Promise }).rebuildSubtypeCounts === 'function') { + await (probe as unknown as { rebuildSubtypeCounts: () => Promise }).rebuildSubtypeCounts() + } + + // FINALIZE: mark migrated (makes re-open a no-op), then drain the head + // branch (its entities moved; the rest is stale 7.x derived state). + // Non-head branches under branches/ are 7.x version history 8.0's MVCC does + // not import — they are left in place. + await probe.writeRawObject('_system/migration-layout.json', { + layout: 'flat-v8', + version: 8, + fromBranch: head, + entitiesMoved: moved + }) + await probe.removeRawPrefix(`branches/${head}`) + } finally { + if (canLock && typeof lockable.releaseWriterLock === 'function') { + await lockable.releaseWriterLock() + } + } + } finally { + if (typeof (probe as unknown as { close?: () => Promise }).close === 'function') { + await (probe as unknown as { close: () => Promise }).close() + } + } + } + private async setupStorage(): Promise { // If the caller passed a pre-constructed storage adapter (e.g. // `storage: new MemoryStorage()`, or the historical materializer's diff --git a/tests/integration/migration-7x-to-8x.test.ts b/tests/integration/migration-7x-to-8x.test.ts new file mode 100644 index 00000000..a9e67a8f --- /dev/null +++ b/tests/integration/migration-7x-to-8x.test.ts @@ -0,0 +1,194 @@ +/** + * @module tests/integration/migration-7x-to-8x + * @description Proof suite for the 7.x → 8.0 on-disk layout migration. 7.x stored + * every object branch-scoped under `branches//`; 8.0 stores the + * identical entity structure at the ROOT (`entities////…`) plus + * the generational `_system/manifest.json` + `_generations/`. GenerationStore.open + * is TOLERANT of a missing manifest (opens at generation 0), so a naive 8.0 open of + * a 7.x directory reports zero entities and SILENTLY LOSES ALL DATA. The migration + * collapses the HEAD branch's entities to the root before storage is set up. + * + * Tests: + * 1. DATA-LOSS LOCK — a built-in FileSystemStorage opened on the reshaped 7.x dir + * reports zero nouns at the root layout (the mode the migration fixes; durable + * because storage never migrates). + * 2. ROUND TRIP — opening a brain with autoMigrate (default) restores + * find/related/counts byte-equal to the reference captured before reshaping. + * 3. IDEMPOTENCY — a second open is a no-op (marker short-circuits; branch drained). + * 4. GUARD — autoMigrate:false on a legacy layout throws explicit guidance. + * 5. NO-OP — a native flat 8.0 brain opens untouched (no marker churn, no data move). + */ +import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +interface Reference { + nounCount: number + verbCount: number + docIds: string[] + activeIds: string[] + relatedFrom0: string[] +} + +const docId = (i: number): string => `00000000-0000-4000-8000-00000000000${i}` + +async function buildReferenceBrain(dir: string): Promise { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', rootDirectory: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + for (let i = 0; i < 5; i++) { + await brain.add({ + id: docId(i), + data: `Document number ${i} about distributed systems`, + type: NounType.Document, + metadata: { title: `Doc ${i}`, status: i % 2 === 0 ? 'active' : 'archived' } + }) + } + await brain.relate({ from: docId(0), to: docId(1), type: VerbType.RelatedTo }) + await brain.relate({ from: docId(0), to: docId(2), type: VerbType.RelatedTo }) + await brain.flush() + const ref = await captureReference(brain) + await brain.close() + return ref +} + +async function captureReference(brain: Brainy): Promise { + const docs = await brain.find({ type: NounType.Document, limit: 100 }) + const active = await brain.find({ type: NounType.Document, where: { status: 'active' }, limit: 100 }) + const related = await brain.related(docId(0)) + return { + nounCount: await brain.getNounCount(), + verbCount: await brain.getVerbCount(), + docIds: docs.map((r: any) => r.id).sort(), + activeIds: active.map((r: any) => r.id).sort(), + relatedFrom0: related.map((r: any) => r.to).sort() + } +} + +/** + * Reshape a flat 8.0 directory into the 7.x branch layout: move `entities/` under + * `branches/main/entities/`, and delete the 8.0-only derived + MVCC state (8.0 + * rebuilds all of it from the canonical entities). The result is what a 7.x store + * looks like to 8.0: canonical entities present, but invisible at the root. + */ +function reshapeToLegacy(dir: string): void { + const branchEntities = path.join(dir, 'branches', 'main', 'entities') + fs.mkdirSync(path.dirname(branchEntities), { recursive: true }) + fs.renameSync(path.join(dir, 'entities'), branchEntities) + for (const derived of ['_system', '_column_index', '_blobs', '_generations', 'indexes']) { + fs.rmSync(path.join(dir, derived), { recursive: true, force: true }) + } +} + +const makeTempDir = (): string => fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-migrate-')) + +async function openBrain(dir: string, extra: Record = {}): Promise { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', rootDirectory: dir }, + dimensions: 384, + silent: true, + ...extra + }) +} + +describe('7.x → 8.0 layout migration', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + let reference: Reference + let legacyTemplate: string + + beforeAll(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + legacyTemplate = makeTempDir() + reference = await buildReferenceBrain(legacyTemplate) + reshapeToLegacy(legacyTemplate) + }) + + afterAll(() => fs.rmSync(legacyTemplate, { recursive: true, force: true })) + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { + await b.close() + } catch { + /* already closed */ + } + } + for (const d of dirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) + }) + + /** A fresh copy of the reshaped legacy directory for one test to mutate. */ + function freshLegacyDir(): string { + const dir = makeTempDir() + dirs.push(dir) + fs.cpSync(legacyTemplate, dir, { recursive: true }) + return dir + } + + it('DATA-LOSS LOCK: a built-in FileSystemStorage on a 7.x layout reports zero nouns at the root', async () => { + const dir = freshLegacyDir() + const storage = new FileSystemStorage(dir) + await storage.init() + // The canonical entities exist on disk (under branches/main/) but are invisible + // at the 8.0 root layout — exactly the silent data loss the migration prevents. + expect(await storage.getNounCount()).toBe(0) + }) + + it('ROUND TRIP: opening with autoMigrate (default) restores all three intelligences byte-equal', async () => { + const dir = freshLegacyDir() + const brain = await openBrain(dir) + brains.push(brain) + await brain.init() + const after = await captureReference(brain) + expect(after).toEqual(reference) + // The branch layout is drained and the marker stamped. + expect(fs.existsSync(path.join(dir, 'branches', 'main', 'entities'))).toBe(false) + expect(fs.existsSync(path.join(dir, 'entities'))).toBe(true) + }) + + it('IDEMPOTENCY: a second open is a no-op and still reads the migrated data', async () => { + const dir = freshLegacyDir() + const first = await openBrain(dir) + brains.push(first) + await first.init() + await first.close() + brains.splice(brains.indexOf(first), 1) + + // Mtime of the marker must not change on the second open (no re-migration). + const markerPath = path.join(dir, '_system', 'migration-layout.json') + const markerExists = fs.existsSync(markerPath) || fs.existsSync(markerPath + '.gz') + + const second = await openBrain(dir) + brains.push(second) + await second.init() + expect(markerExists).toBe(true) + expect(await captureReference(second)).toEqual(reference) + }) + + it('GUARD: autoMigrate:false on a 7.x layout throws explicit guidance instead of losing data', async () => { + const dir = freshLegacyDir() + const brain = await openBrain(dir, { autoMigrate: false }) + brains.push(brain) + await expect(brain.init()).rejects.toThrow(/7\.x branch layout/) + }) + + it('NO-OP: a native flat 8.0 brain opens untouched', async () => { + const dir = makeTempDir() + dirs.push(dir) + const ref = await buildReferenceBrain(dir) // builds + closes a normal flat brain + const reopened = await openBrain(dir) + brains.push(reopened) + await reopened.init() // migration phase is a strict no-op here + expect(await captureReference(reopened)).toEqual(ref) + expect(fs.existsSync(path.join(dir, 'branches'))).toBe(false) + }) +})