diff --git a/RELEASES.md b/RELEASES.md index 58b20bd2..81ef906a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,29 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so --- +## v7.31.8 — 2026-06-16 + +**Affected products:** every consumer on a bare VM or `mmap-filesystem` storage — **upgrade recommended** (Memory, Workshop on 7.31.x; Venue on 7.28.x). Two platform-wide production fixes. Drop-in; no API or data changes. + +### Fix 1 — auto query-limit cap no longer collapses on a healthy VM (the 500s) + +`new Brainy()`'s auto-configured `maxQueryLimit` could collapse to ~1000 on a perfectly healthy box, throwing `exceeds the auto-configured query limit` on legitimate queries (e.g. `find({ limit: 2400 })`) → HTTP 500s. Root cause: detection read `os.freemem()` (kernel MemFree), which **excludes reclaimable page cache** — on a long-running box using mmap-filesystem storage the page cache grows until MemFree is a thin sliver (tens of MB) even though several GB are actually available, so `floor(freemem / 25KB)·1000` floored the cap to ~1000. + +- Memory detection now reads **`/proc/meminfo` `MemAvailable`** (the figure `free -h` shows — counts reclaimable cache), falling back to `os.freemem()` only off-Linux. +- A **10,000 floor** on auto-detected caps (container / free-memory branches) so a detection miss can never silently collapse the cap. Explicit `maxQueryLimit` / `reservedQueryMemory` settings are honored as-is (never floored). + +No action beyond upgrading. A `MEMORY_LIMIT` override (if you set one as a workaround) is still honored. + +### Fix 2 — native mmap vector fast-path re-engages (cold-start 57s → instant) + +`FileSystemStorage` now exposes a public `rootDirectory` getter. The optional native vector provider feature-detects `storage.rootDirectory` to enable its memory-mapped graph fast path; brainy stored the path as the protected `rootDir`, so the gate silently failed and every cold start rebuilt the index via per-entity disk reads (~57s for a 283 MB brain on a measured deployment). With the getter, the mmap path engages and cold start becomes a near-instant memory-mapped load. + +Self-heals after the first post-upgrade flush writes the mmap file — the first restart after upgrading rebuilds once, then every subsequent boot is fast. Benefits the full deployed matrix (the gate exists in the native provider versions Memory/Workshop and Venue run). + +**Upgrade:** bump `@soulcraft/brainy` to `7.31.8`. No code or data migration. + +--- + ## v7.31.2 — 2026-06-09 **Affected products:** none in behaviour; affects anyone reading Brainy's TypeScript diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index e86e1a6d..3b11cf65 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -81,6 +81,19 @@ export class FileSystemStorage extends BaseStorage { private readonly MAX_SHARDS = 256 // Hex range: 00-ff private cachedShardingDepth: number = this.SHARDING_DEPTH // Always use fixed depth protected rootDir: string + + /** + * Public read-only alias for the storage root path. Native vector plugins + * (e.g. `@soulcraft/cortex`) feature-detect `storage.rootDirectory` to enable + * their memory-mapped vector fast path; brainy stores it as the protected + * `rootDir`, so without this getter the native mmap backend stays un-wired and + * the index rebuilds via slow per-entity disk reads on cold start. Exposing + * the alias lets the native backend memory-map its index file instead. + */ + get rootDirectory(): string { + return this.rootDir + } + private nounsDir!: string private verbsDir!: string private metadataDir!: string diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index cb26baaf..86b06ea5 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -32,6 +32,24 @@ const getSystemMemory = (): number => { } const getAvailableMemory = (): number => { + // Prefer /proc/meminfo `MemAvailable` — it counts reclaimable page cache as + // available, which `os.freemem()` (kernel MemFree) does NOT. On a long-running + // box using mmap-filesystem storage, the page cache grows until MemFree is a + // thin sliver (tens of MB) even though several GB are actually available; the + // freemem reading then collapses the auto query-limit cap to ~1000 and 500s + // legitimate queries. MemAvailable reflects the real figure `free -h` shows. + if (fs) { + try { + const meminfo = fs.readFileSync('/proc/meminfo', 'utf8') as string + const match = meminfo.match(/^MemAvailable:\s+(\d+)\s*kB/m) + if (match) { + const bytes = parseInt(match[1], 10) * 1024 + if (bytes > 0) return bytes + } + } catch { + // Not Linux / no procfs — fall through to freemem. + } + } if (os) { return os.freemem() } @@ -137,6 +155,11 @@ const getContainerMemoryLimit = (): number | null => { * the throw tier still fires before OOM territory (72 K+). */ const MAX_LIMIT_KB_PER_RESULT = 25 +// Floor for AUTO-DETECTED caps (container/free-memory branches). A memory +// misread must never silently collapse the cap below a practical safety value +// and 500 legitimate queries; explicit operator settings (maxQueryLimit / +// reservedQueryMemory) are honored as-is and bypass this floor. +const MIN_AUTO_QUERY_LIMIT = 10000 /** * One-time-per-call-site warning dedup. Keyed on the caller location returned @@ -230,9 +253,12 @@ export class ValidationConfig { // Reserve 25% for query operations const queryMemory = this.detectedContainerLimit * 0.25 - this.maxLimit = Math.min( - 100000, - Math.floor(queryMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000 + this.maxLimit = Math.max( + MIN_AUTO_QUERY_LIMIT, + Math.min( + 100000, + Math.floor(queryMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000 + ) ) this.limitBasis = 'containerMemory' @@ -246,9 +272,12 @@ export class ValidationConfig { // Priority 4: Free memory (fallback, current behavior) const availableMemory = getAvailableMemory() - this.maxLimit = Math.min( - 100000, - Math.floor(availableMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000 + this.maxLimit = Math.max( + MIN_AUTO_QUERY_LIMIT, + Math.min( + 100000, + Math.floor(availableMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000 + ) ) this.limitBasis = 'freeMemory' diff --git a/tests/unit/storage/fileSystemStorage-rootDirectory.test.ts b/tests/unit/storage/fileSystemStorage-rootDirectory.test.ts new file mode 100644 index 00000000..83b0c2f5 --- /dev/null +++ b/tests/unit/storage/fileSystemStorage-rootDirectory.test.ts @@ -0,0 +1,28 @@ +/** + * Regression: FileSystemStorage must expose a public `rootDirectory` getter. + * + * Native vector plugins (e.g. @soulcraft/cortex's NativeHNSWWrapper) feature-detect + * `storage.rootDirectory` to enable their memory-mapped vector fast path. Brainy + * stores the path as the protected `rootDir`; without this public alias the native + * mmap backend stayed un-wired and the index rebuilt via slow per-entity disk reads + * on cold start (BRAINY-MMAP-VECTOR-HOOK). The getter is the load-bearing line, so + * guard it explicitly. + */ +import { describe, it, expect } from 'vitest' +import { mkdtempSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' + +describe('FileSystemStorage.rootDirectory (native mmap-backend hook)', () => { + it('exposes the constructor root path via a public rootDirectory getter', () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-rootdir-')) + try { + const storage = new FileSystemStorage(dir) + // The exact property a native vector backend reads to engage its mmap path. + expect((storage as unknown as { rootDirectory: string }).rootDirectory).toBe(dir) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/unit/utils/memoryLimits.test.ts b/tests/unit/utils/memoryLimits.test.ts index aca6d91c..e473903d 100644 --- a/tests/unit/utils/memoryLimits.test.ts +++ b/tests/unit/utils/memoryLimits.test.ts @@ -399,4 +399,32 @@ describe('Memory Limits - Container Detection & Smart Calculation', () => { await brain.close() }) }) + + // Regression: a memory MISREAD (tiny detected limit, or os.freemem reading a + // sliver on a page-cache-heavy mmap box) must NOT collapse the auto cap to + // ~1000 and 500 legitimate queries. Auto-detected caps are floored at 10k; + // explicit operator settings still pass through unfloored. + describe('Auto-cap floor (BUG: querycap collapse on healthy VM)', () => { + it('floors the container-derived cap so a tiny/misread limit cannot collapse it', () => { + // 50 MB → queryMemory 12.5 MB → floor(12.5/25)·1000 = 0 pre-fix. + process.env.MEMORY_LIMIT = String(50 * 1024 * 1024) + const config = ValidationConfig.getInstance() + expect(config.limitBasis).toBe('containerMemory') + expect(config.maxLimit).toBe(10000) // floored, not 0/1000 + }) + + it('floors the free-memory-derived cap as well', () => { + // No container env → freeMemory branch. On any real box this is well + // above the floor; the guarantee is it can never drop below it. + const config = ValidationConfig.getInstance() + expect(config.limitBasis).toBe('freeMemory') + expect(config.maxLimit).toBeGreaterThanOrEqual(10000) + }) + + it('does NOT floor an explicit (small) operator override', () => { + const config = ValidationConfig.getInstance({ maxQueryLimit: 500 }) + expect(config.limitBasis).toBe('override') + expect(config.maxLimit).toBe(500) // explicit intent respected, not floored + }) + }) })