diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index dd0445ff..81390eab 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -265,8 +265,19 @@ export class FileSystemStorage extends BaseStorage { console.log(`✅ Migration complete - now using depth ${this.SHARDING_DEPTH} sharding`) } else if (detectedDepth === null) { - // New installation - console.log(`📁 New installation: using depth ${this.SHARDING_DEPTH} sharding (optimal for 1-2.5M entities)`) + // The legacy sharding probe inspects `entities/nouns/hnsw` — a 7.x + // directory the 8.0 write path never populates (8.0 stores entities in + // the canonical `entities/nouns///` layout), so it returns + // null for EVERY 8.0 store, new or not. Decide new-vs-existing from the + // canonical layout the DB actually reads/writes so an established brain + // is not mislabeled "New installation" on every boot. + const established = + this.totalNounCount > 0 || (await this.hasCanonicalEntities()) + console.log( + established + ? `📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)` + : `📁 New installation: using depth ${this.SHARDING_DEPTH} sharding (optimal for 1-2.5M entities)` + ) } else { // Already using correct depth console.log(`📁 Using depth ${this.SHARDING_DEPTH} sharding (${this.totalNounCount} entities)`) @@ -2737,6 +2748,39 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * Whether this store already holds canonical 8.0 entities. + * + * 8.0 writes nouns to `entities/nouns///vectors.json` (see + * `getNounVectorPath`), but the legacy sharding probe + * ({@link detectExistingShardingDepth}) inspects `entities/nouns/hnsw` — a 7.x + * directory the 8.0 write path never populates. The probe therefore returns + * null for every 8.0 store and cannot tell an established brain from a fresh + * one, which mislabels established stores "New installation" on every boot. + * This checks the canonical shard tree the DB actually reads and writes (the + * same `entities/nouns/` shards `getNounsWithPagination` walks) so boot + * logs are truthful. + * + * @returns true if at least one 2-hex shard directory (00–ff) exists under + * `entities/nouns/`, i.e. the store has previously persisted entities. + */ + private async hasCanonicalEntities(): Promise { + const canonicalNounsDir = path.join(this.rootDir, 'entities', 'nouns') + try { + const entries = await fs.promises.readdir(canonicalNounsDir, { + withFileTypes: true + }) + // A populated store has ≥1 two-hex shard dir (00–ff). The vestigial + // `hnsw` subdir is 4 chars and correctly excluded by the hex test. + return entries.some( + (e: any) => e.isDirectory() && /^[0-9a-f]{2}$/i.test(e.name) + ) + } catch { + // Directory absent (fresh store) or unreadable → no persisted entities. + return false + } + } + /** * Get sharding depth * Always returns 1 (single-level sharding) for optimal balance of diff --git a/tests/unit/storage/sharding-new-installation-log.test.ts b/tests/unit/storage/sharding-new-installation-log.test.ts new file mode 100644 index 00000000..ca674137 --- /dev/null +++ b/tests/unit/storage/sharding-new-installation-log.test.ts @@ -0,0 +1,105 @@ +/** + * @module tests/unit/storage/sharding-new-installation-log + * @description Regression for BR-8-BOOT-INDEX symptom 1. Brainy 8.0 stores + * entities in the canonical `entities/nouns///vectors.json` layout, + * but the legacy sharding probe (`detectExistingShardingDepth`) inspected + * `entities/nouns/hnsw` — a 7.x directory the 8.0 write path never populates — + * so it returned null and logged "📁 New installation" for EVERY store on EVERY + * boot, including established brains holding thousands of entities. The + * new-vs-existing decision now consults the canonical layout the DB actually + * uses (`hasCanonicalEntities`), so an established brain reports its true entity + * count and only a genuinely empty store says "New installation". + */ +import { describe, it, expect, vi, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import type { NounMetadata } from '../../../src/coreTypes.js' + +const dirs: string[] = [] +function mkTmp(): string { + const d = mkdtempSync(join(tmpdir(), 'brainy-newinstall-')) + dirs.push(d) + return d +} +afterEach(() => { + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) + vi.restoreAllMocks() +}) + +/** Release a raw FileSystemStorage so a subsequent open of the dir is unblocked. */ +async function teardown(s: any): Promise { + try { await s.flush?.() } catch { /* best effort */ } + try { s.stopFlushRequestWatcher?.() } catch { /* best effort */ } + try { await s.releaseWriterLock?.() } catch { /* best effort */ } +} + +/** Capture everything written to console.log during `fn()`. */ +async function captureLog(fn: () => Promise): Promise { + const lines: string[] = [] + const spy = vi + .spyOn(console, 'log') + .mockImplementation((...args: any[]) => { + lines.push(args.map(String).join(' ')) + }) + try { + await fn() + } finally { + spy.mockRestore() + } + return lines.join('\n') +} + +const VEC = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] +async function seedOne(s: any, id: string): Promise { + await s.saveNounMetadata(id, { + noun: 'thing', + createdAt: Date.now(), + updatedAt: Date.now() + } as NounMetadata) + await s.saveNoun({ id, vector: VEC, connections: new Map(), level: 0 }) +} + +describe('FileSystemStorage boot log: an established brain is not "New installation" (BR-8-BOOT-INDEX)', () => { + it('a genuinely fresh store still logs "New installation"', async () => { + const dir = mkTmp() + const storage = new FileSystemStorage(dir) + const out = await captureLog(() => storage.init()) + expect(out).toContain('New installation') + await teardown(storage) + }) + + it('an established store (canonical entities present) does NOT log "New installation" on reopen', async () => { + const dir = mkTmp() + + // 1. Author an entity → canonical entities/nouns///vectors.json. + const first = new FileSystemStorage(dir) + await first.init() + await seedOne(first, 'a'.repeat(32)) + await teardown(first) + + // 2. Reopen: the boot log must reflect the established store, not "New installation". + const second = new FileSystemStorage(dir) + const out = await captureLog(() => second.init()) + expect(out).not.toContain('New installation') + expect(out).toContain('Using depth 1 sharding') + await teardown(second) + }) + + it('hasCanonicalEntities() sees the canonical shard tree — independent of counts.json', async () => { + const dir = mkTmp() + const s: any = new FileSystemStorage(dir) + await s.init() + + // Fresh: only the vestigial `hnsw` dir exists → no canonical entities. + expect(await s.hasCanonicalEntities()).toBe(false) + + // After a canonical write the 2-hex shard dir exists → established, even if + // counts.json were later lost on a container restart (the fallback path). + await seedOne(s, 'c'.repeat(32)) + expect(await s.hasCanonicalEntities()).toBe(true) + + await teardown(s) + }) +})