fix: query-cap memory misread (MemAvailable + floor) + rootDirectory getter for native mmap fast-path
Two platform-wide production fixes for consumers on bare VMs / mmap-filesystem storage. BUG A — auto maxQueryLimit collapsed to ~1000 on healthy VMs → 500s on legitimate queries. getAvailableMemory() read os.freemem() (kernel MemFree, which excludes reclaimable page cache and reads as tens of MB on a page-cache-heavy mmap box). Now reads /proc/meminfo MemAvailable (the `free -h` figure), falling back to os.freemem() only off-Linux; auto-detected caps (container/free branches) are floored at 10k so a misread can't collapse them. Explicit maxQueryLimit/reservedQueryMemory are honored as-is. BUG B — native mmap vector fast-path never engaged (per-entity reads → 57s cold start on a 283MB brain). FileSystemStorage now exposes a public `rootDirectory` getter; the native vector provider feature-detects it to enable its memory-mapped graph path. brainy stored it as the protected `rootDir`, so the gate silently failed. Self-heals after the first post-upgrade flush writes the mmap file. Regression tests: auto-cap floor (container/free/explicit-override) + the rootDirectory getter. Build clean; full suite 1483 green.
This commit is contained in:
parent
4f8159c572
commit
3f8e0971a2
5 changed files with 127 additions and 6 deletions
|
|
@ -81,6 +81,19 @@ export class FileSystemStorage extends BaseStorage {
|
|||
private readonly MAX_SHARDS = 256 // Hex range: 00-ff
|
||||
private cachedShardingDepth: number = this.SHARDING_DEPTH // Always use fixed depth
|
||||
protected rootDir: string
|
||||
|
||||
/**
|
||||
* Public read-only alias for the storage root path. Native vector plugins
|
||||
* (e.g. `@soulcraft/cortex`) feature-detect `storage.rootDirectory` to enable
|
||||
* their memory-mapped vector fast path; brainy stores it as the protected
|
||||
* `rootDir`, so without this getter the native mmap backend stays un-wired and
|
||||
* the index rebuilds via slow per-entity disk reads on cold start. Exposing
|
||||
* the alias lets the native backend memory-map its index file instead.
|
||||
*/
|
||||
get rootDirectory(): string {
|
||||
return this.rootDir
|
||||
}
|
||||
|
||||
private nounsDir!: string
|
||||
private verbsDir!: string
|
||||
private metadataDir!: string
|
||||
|
|
|
|||
|
|
@ -32,6 +32,24 @@ const getSystemMemory = (): number => {
|
|||
}
|
||||
|
||||
const getAvailableMemory = (): number => {
|
||||
// Prefer /proc/meminfo `MemAvailable` — it counts reclaimable page cache as
|
||||
// available, which `os.freemem()` (kernel MemFree) does NOT. On a long-running
|
||||
// box using mmap-filesystem storage, the page cache grows until MemFree is a
|
||||
// thin sliver (tens of MB) even though several GB are actually available; the
|
||||
// freemem reading then collapses the auto query-limit cap to ~1000 and 500s
|
||||
// legitimate queries. MemAvailable reflects the real figure `free -h` shows.
|
||||
if (fs) {
|
||||
try {
|
||||
const meminfo = fs.readFileSync('/proc/meminfo', 'utf8') as string
|
||||
const match = meminfo.match(/^MemAvailable:\s+(\d+)\s*kB/m)
|
||||
if (match) {
|
||||
const bytes = parseInt(match[1], 10) * 1024
|
||||
if (bytes > 0) return bytes
|
||||
}
|
||||
} catch {
|
||||
// Not Linux / no procfs — fall through to freemem.
|
||||
}
|
||||
}
|
||||
if (os) {
|
||||
return os.freemem()
|
||||
}
|
||||
|
|
@ -137,6 +155,11 @@ const getContainerMemoryLimit = (): number | null => {
|
|||
* the throw tier still fires before OOM territory (72 K+).
|
||||
*/
|
||||
const MAX_LIMIT_KB_PER_RESULT = 25
|
||||
// Floor for AUTO-DETECTED caps (container/free-memory branches). A memory
|
||||
// misread must never silently collapse the cap below a practical safety value
|
||||
// and 500 legitimate queries; explicit operator settings (maxQueryLimit /
|
||||
// reservedQueryMemory) are honored as-is and bypass this floor.
|
||||
const MIN_AUTO_QUERY_LIMIT = 10000
|
||||
|
||||
/**
|
||||
* One-time-per-call-site warning dedup. Keyed on the caller location returned
|
||||
|
|
@ -230,9 +253,12 @@ export class ValidationConfig {
|
|||
// Reserve 25% for query operations
|
||||
const queryMemory = this.detectedContainerLimit * 0.25
|
||||
|
||||
this.maxLimit = Math.min(
|
||||
100000,
|
||||
Math.floor(queryMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000
|
||||
this.maxLimit = Math.max(
|
||||
MIN_AUTO_QUERY_LIMIT,
|
||||
Math.min(
|
||||
100000,
|
||||
Math.floor(queryMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000
|
||||
)
|
||||
)
|
||||
this.limitBasis = 'containerMemory'
|
||||
|
||||
|
|
@ -246,9 +272,12 @@ export class ValidationConfig {
|
|||
// Priority 4: Free memory (fallback, current behavior)
|
||||
const availableMemory = getAvailableMemory()
|
||||
|
||||
this.maxLimit = Math.min(
|
||||
100000,
|
||||
Math.floor(availableMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000
|
||||
this.maxLimit = Math.max(
|
||||
MIN_AUTO_QUERY_LIMIT,
|
||||
Math.min(
|
||||
100000,
|
||||
Math.floor(availableMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000
|
||||
)
|
||||
)
|
||||
this.limitBasis = 'freeMemory'
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue