fix: clear() wipes the full native/derived footprint, not a subset
clear() removed entities, indexes, system and _cas but left three top-level trees on disk: _blobs (raw HNSW/LSM segment bytes and the native dkann index), _id_mapper (the native shared mmap id-mapper), and _column_index (column-store manifests). A cleared brain therefore re-read stale native state. Worse, the column store splits its state across two of those trees — manifests under _column_index/ and their segment bytes under _blobs/_column_index/ — so removing one without the other stranded a manifest listing segments that no longer exist, which the hardened segment-load path now (correctly) refuses with ColumnSegmentLoadError. The three trees now fall together as a set, exactly as _cas already does. locks/ is deliberately preserved: it is live coordination state, not data.
This commit is contained in:
parent
af5d2f389b
commit
d8301f8d08
2 changed files with 92 additions and 0 deletions
74
tests/unit/storage/clear-native-footprint.test.ts
Normal file
74
tests/unit/storage/clear-native-footprint.test.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
/**
|
||||
* @module tests/unit/storage/clear-native-footprint
|
||||
* @description clear() must remove the COMPLETE derived/native on-disk footprint
|
||||
* (finding 7). Before the fix it wiped only entities/indexes/system/_cas and left
|
||||
* three top-level trees behind:
|
||||
* - `_blobs` raw binary blobs (HNSW/LSM segments, the native dkann index,
|
||||
* and column-store segment bytes under `_blobs/_column_index/`)
|
||||
* - `_id_mapper` the native shared mmap id-mapper
|
||||
* - `_column_index` the column-store MANIFEST.json files
|
||||
* A cleared brain therefore re-read stale native state, and — worst — an orphaned
|
||||
* column manifest whose segment bytes had been removed with `_blobs`, which the
|
||||
* hardened load path now refuses with ColumnSegmentLoadError. The three must fall
|
||||
* together as a set.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
|
||||
|
||||
const exists = (p: string): boolean => fs.existsSync(p)
|
||||
|
||||
describe('FileSystemStorage.clear() wipes the full native/derived footprint (finding 7)', () => {
|
||||
let dir: string
|
||||
let storage: any
|
||||
|
||||
beforeEach(async () => {
|
||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-clear-footprint-'))
|
||||
storage = new FileSystemStorage(dir)
|
||||
await storage.init()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('removes _blobs, _id_mapper and _column_index (previously left stranded)', async () => {
|
||||
// A raw blob (this is how HNSW/LSM/column segments persist) → creates _blobs.
|
||||
await storage.saveBinaryBlob('graph-lsm/source/sstable-1', Buffer.from([1, 2, 3, 4]))
|
||||
// A column segment blob lives UNDER _blobs/_column_index/...
|
||||
await storage.saveBinaryBlob('_column_index/createdAt/L0-000000', Buffer.from([5, 6, 7, 8]))
|
||||
|
||||
// Simulate the native id-mapper mmap dir and the column manifest object-tree.
|
||||
const idMapperDir = path.join(dir, '_id_mapper')
|
||||
fs.mkdirSync(idMapperDir, { recursive: true })
|
||||
fs.writeFileSync(path.join(idMapperDir, 'main.slotmap'), Buffer.from([0, 1]))
|
||||
const colDir = path.join(dir, '_column_index', 'createdAt')
|
||||
fs.mkdirSync(colDir, { recursive: true })
|
||||
fs.writeFileSync(
|
||||
path.join(colDir, 'MANIFEST.json'),
|
||||
JSON.stringify({ segments: [{ id: 0, level: 0, count: 1 }] })
|
||||
)
|
||||
|
||||
// Sanity: everything is present before the clear.
|
||||
expect(exists(path.join(dir, '_blobs'))).toBe(true)
|
||||
expect(exists(idMapperDir)).toBe(true)
|
||||
expect(exists(path.join(dir, '_column_index'))).toBe(true)
|
||||
|
||||
await storage.clear()
|
||||
|
||||
// The complete derived footprint is gone — no stale native state, and no
|
||||
// orphaned column manifest pointing at removed segment bytes.
|
||||
expect(exists(path.join(dir, '_blobs'))).toBe(false)
|
||||
expect(exists(idMapperDir)).toBe(false)
|
||||
expect(exists(path.join(dir, '_column_index'))).toBe(false)
|
||||
})
|
||||
|
||||
it('clear() is a no-op-safe when the native dirs never existed', async () => {
|
||||
// Fresh store, no blobs written — clear must not throw on absent dirs.
|
||||
await expect(storage.clear()).resolves.toBeUndefined()
|
||||
expect(exists(path.join(dir, '_blobs'))).toBe(false)
|
||||
expect(exists(path.join(dir, '_column_index'))).toBe(false)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue