diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 27f1af89..7e341ac5 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -1349,6 +1349,24 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.rm(casDir, { recursive: true, force: true }) } + // Remove the raw-blob + native-shared + column-index footprint. These + // top-level trees were NOT wiped before, so a cleared brain re-read stale + // native blobs (HNSW/LSM segments, the native dkann index), a stale native + // id-mapper, and — worst — orphaned column manifests. `_column_index` holds + // the column-store MANIFEST.json files while their segment bytes live under + // `_blobs/_column_index/...`; removing one without the other would strand a + // manifest listing segments that no longer exist, which the load path now + // (correctly) refuses with ColumnSegmentLoadError. They must fall together, + // as a set, exactly as `_cas` does — the complete derived footprint, not a + // subset. (`locks/` is deliberately left: it is live coordination state, + // not data.) + for (const nativeDir of ['_blobs', '_id_mapper', '_column_index']) { + const dir = path.join(this.rootDir, nativeDir) + if (await this.directoryExists(dir)) { + await fs.promises.rm(dir, { recursive: true, force: true }) + } + } + // Reset ALL shared derived state via the same path restore uses: the // write-through cache, id→type/subtype caches, per-type/subtype count // rollups, statistics cache, graph-index singleton, and the BlobStorage diff --git a/tests/unit/storage/clear-native-footprint.test.ts b/tests/unit/storage/clear-native-footprint.test.ts new file mode 100644 index 00000000..313ed9b8 --- /dev/null +++ b/tests/unit/storage/clear-native-footprint.test.ts @@ -0,0 +1,74 @@ +/** + * @module tests/unit/storage/clear-native-footprint + * @description clear() must remove the COMPLETE derived/native on-disk footprint + * (finding 7). Before the fix it wiped only entities/indexes/system/_cas and left + * three top-level trees behind: + * - `_blobs` raw binary blobs (HNSW/LSM segments, the native dkann index, + * and column-store segment bytes under `_blobs/_column_index/`) + * - `_id_mapper` the native shared mmap id-mapper + * - `_column_index` the column-store MANIFEST.json files + * A cleared brain therefore re-read stale native state, and — worst — an orphaned + * column manifest whose segment bytes had been removed with `_blobs`, which the + * hardened load path now refuses with ColumnSegmentLoadError. The three must fall + * together as a set. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' + +const exists = (p: string): boolean => fs.existsSync(p) + +describe('FileSystemStorage.clear() wipes the full native/derived footprint (finding 7)', () => { + let dir: string + let storage: any + + beforeEach(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-clear-footprint-')) + storage = new FileSystemStorage(dir) + await storage.init() + }) + + afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('removes _blobs, _id_mapper and _column_index (previously left stranded)', async () => { + // A raw blob (this is how HNSW/LSM/column segments persist) → creates _blobs. + await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([1, 2, 3, 4])) + // A column segment blob lives UNDER _blobs/_column_index/... + await storage.saveBinaryBlob('_column_index/createdAt/L0-000000', Buffer.from([5, 6, 7, 8])) + + // Simulate the native id-mapper mmap dir and the column manifest object-tree. + const idMapperDir = path.join(dir, '_id_mapper') + fs.mkdirSync(idMapperDir, { recursive: true }) + fs.writeFileSync(path.join(idMapperDir, 'main.slotmap'), Buffer.from([0, 1])) + const colDir = path.join(dir, '_column_index', 'createdAt') + fs.mkdirSync(colDir, { recursive: true }) + fs.writeFileSync( + path.join(colDir, 'MANIFEST.json'), + JSON.stringify({ segments: [{ id: 0, level: 0, count: 1 }] }) + ) + + // Sanity: everything is present before the clear. + expect(exists(path.join(dir, '_blobs'))).toBe(true) + expect(exists(idMapperDir)).toBe(true) + expect(exists(path.join(dir, '_column_index'))).toBe(true) + + await storage.clear() + + // The complete derived footprint is gone — no stale native state, and no + // orphaned column manifest pointing at removed segment bytes. + expect(exists(path.join(dir, '_blobs'))).toBe(false) + expect(exists(idMapperDir)).toBe(false) + expect(exists(path.join(dir, '_column_index'))).toBe(false) + }) + + it('clear() is a no-op-safe when the native dirs never existed', async () => { + // Fresh store, no blobs written — clear must not throw on absent dirs. + await expect(storage.clear()).resolves.toBeUndefined() + expect(exists(path.join(dir, '_blobs'))).toBe(false) + expect(exists(path.join(dir, '_column_index'))).toBe(false) + }) +})