feat: OS-limit detection for pool-scale deployments
- New src/utils/osLimits.ts: reads RLIMIT_NOFILE (soft/hard, from /proc/self/limits) and vm.max_map_count at open — once per process, Linux-only, measurement-only — and warns loudly when either sits below the pool-scale floors (soft NOFILE < 65536, max_map_count < 262144), with the exact raise commands. On stock defaults the failure otherwise arrives as EMFILE or a failed mmap deep inside an index open, long after the cause stopped being visible. An unreadable limit produces NO warning — no measurement, no claim — so non-Linux platforms stay silent. - Exported for ops doors: checkOsLimits() returns the full OsLimitsReport; floors exported as constants. Wired fire-and-forget in performInit after storage init; the check can never affect open. - Unit tests pin the parser (incl. 'unlimited'), the floor thresholds, the null-never-warns rule, and the off-Linux silent path.
This commit is contained in:
parent
dcd5036fe9
commit
16a73b8475
5 changed files with 246 additions and 0 deletions
141
src/utils/osLimits.ts
Normal file
141
src/utils/osLimits.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
/**
|
||||
* @module utils/osLimits
|
||||
* @description Detect-and-warn for OS resource limits that bite at POOL scale.
|
||||
*
|
||||
* A single brain rarely notices them, but a pool of brains — especially with a
|
||||
* native accelerator memory-mapping many index files per brain — consumes file
|
||||
* descriptors and memory mappings multiplicatively. On stock Linux defaults
|
||||
* (RLIMIT_NOFILE soft 1024, vm.max_map_count 65530) the failure arrives as
|
||||
* EMFILE or a failed mmap deep inside an index open, long after the real cause
|
||||
* (the limit) stopped being visible. This module reads the limits at open and
|
||||
* WARNS ONCE per process with the exact raise commands, so the operator learns
|
||||
* the fix before the incident instead of from it.
|
||||
*
|
||||
* Read-only and Linux-only by construction: both sources are `/proc` files.
|
||||
* On platforms where they are absent the check reports nulls and stays silent —
|
||||
* no limit read means no claim made, never a guessed warning.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs'
|
||||
import { prodLog } from './logger.js'
|
||||
|
||||
/** Soft-NOFILE floor below which pool-scale use is at EMFILE risk. */
|
||||
export const NOFILE_POOL_FLOOR = 65536
|
||||
|
||||
/** vm.max_map_count floor below which mmap-heavy native indexes are at risk. */
|
||||
export const MAX_MAP_COUNT_POOL_FLOOR = 262144
|
||||
|
||||
export interface OsLimitsReport {
|
||||
/** RLIMIT_NOFILE soft limit (null when unreadable; Infinity for 'unlimited'). */
|
||||
nofileSoft: number | null
|
||||
/** RLIMIT_NOFILE hard limit (null when unreadable; Infinity for 'unlimited'). */
|
||||
nofileHard: number | null
|
||||
/** vm.max_map_count (null when unreadable). */
|
||||
maxMapCount: number | null
|
||||
/** Human-actionable warnings for limits below the pool floors. Empty = fine. */
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the `Max open files` row of a `/proc/<pid>/limits` document into
|
||||
* soft/hard values. Returns nulls when the row is absent or malformed.
|
||||
*/
|
||||
export function parseProcLimits(content: string): { soft: number | null; hard: number | null } {
|
||||
const line = content.split('\n').find((l) => l.startsWith('Max open files'))
|
||||
if (!line) return { soft: null, hard: null }
|
||||
const m = line.match(/^Max open files\s+(\S+)\s+(\S+)/)
|
||||
if (!m) return { soft: null, hard: null }
|
||||
const parse = (v: string): number | null => {
|
||||
if (v === 'unlimited') return Infinity
|
||||
const n = Number.parseInt(v, 10)
|
||||
return Number.isNaN(n) ? null : n
|
||||
}
|
||||
return { soft: parse(m[1]), hard: parse(m[2]) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Assess readable limits against the pool floors. Pure — feed it any values.
|
||||
* A null (unreadable) limit produces NO warning: no measurement, no claim.
|
||||
*/
|
||||
export function assessOsLimits(limits: {
|
||||
nofileSoft: number | null
|
||||
nofileHard: number | null
|
||||
maxMapCount: number | null
|
||||
}): string[] {
|
||||
const warnings: string[] = []
|
||||
|
||||
if (limits.nofileSoft !== null && limits.nofileSoft < NOFILE_POOL_FLOOR) {
|
||||
const hardNote =
|
||||
limits.nofileHard !== null && limits.nofileHard >= NOFILE_POOL_FLOOR
|
||||
? ` (the hard limit ${limits.nofileHard === Infinity ? 'unlimited' : limits.nofileHard} already allows it — raise the soft limit only)`
|
||||
: ''
|
||||
warnings.push(
|
||||
`RLIMIT_NOFILE soft limit is ${limits.nofileSoft} — below the ${NOFILE_POOL_FLOOR} recommended ` +
|
||||
`for pool-scale use (a pool of brains with a native accelerator opens many index files per brain; ` +
|
||||
`the failure mode is EMFILE deep inside an index open). Raise with \`ulimit -n ${NOFILE_POOL_FLOOR}\` ` +
|
||||
`or LimitNOFILE=${NOFILE_POOL_FLOOR} in the service unit${hardNote}.`
|
||||
)
|
||||
}
|
||||
|
||||
if (limits.maxMapCount !== null && limits.maxMapCount < MAX_MAP_COUNT_POOL_FLOOR) {
|
||||
warnings.push(
|
||||
`vm.max_map_count is ${limits.maxMapCount} — below the ${MAX_MAP_COUNT_POOL_FLOOR} recommended ` +
|
||||
`for mmap-heavy native indexes at pool scale (each mapped index segment consumes map entries; ` +
|
||||
`the failure mode is a failed mmap mid-heal). Raise with ` +
|
||||
`\`sysctl -w vm.max_map_count=${MAX_MAP_COUNT_POOL_FLOOR}\` (persist in /etc/sysctl.d/).`
|
||||
)
|
||||
}
|
||||
|
||||
return warnings
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the limits from /proc and assess them. `readFile` is injectable for
|
||||
* tests; absent/unreadable sources yield nulls (and therefore no warnings).
|
||||
*/
|
||||
export async function checkOsLimits(
|
||||
readFile: (path: string) => Promise<string> = async (p) => fs.promises.readFile(p, 'utf-8')
|
||||
): Promise<OsLimitsReport> {
|
||||
let nofileSoft: number | null = null
|
||||
let nofileHard: number | null = null
|
||||
let maxMapCount: number | null = null
|
||||
|
||||
try {
|
||||
const parsed = parseProcLimits(await readFile('/proc/self/limits'))
|
||||
nofileSoft = parsed.soft
|
||||
nofileHard = parsed.hard
|
||||
} catch {
|
||||
// Not Linux (or /proc unavailable) — no measurement, no claim.
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = (await readFile('/proc/sys/vm/max_map_count')).trim()
|
||||
const n = Number.parseInt(raw, 10)
|
||||
maxMapCount = Number.isNaN(n) ? null : n
|
||||
} catch {
|
||||
// Not Linux — same rule.
|
||||
}
|
||||
|
||||
const warnings = assessOsLimits({ nofileSoft, nofileHard, maxMapCount })
|
||||
return { nofileSoft, nofileHard, maxMapCount, warnings }
|
||||
}
|
||||
|
||||
/** Once-per-process latch so a brain pool warns once, not once per brain. */
|
||||
let osLimitsWarned = false
|
||||
|
||||
/**
|
||||
* Run the check and warn (once per process) about limits below the pool
|
||||
* floors. Called from brain open; safe everywhere (silent off-Linux).
|
||||
*/
|
||||
export async function warnOnLowOsLimits(): Promise<void> {
|
||||
if (osLimitsWarned) return
|
||||
osLimitsWarned = true
|
||||
try {
|
||||
const report = await checkOsLimits()
|
||||
for (const warning of report.warnings) {
|
||||
prodLog.warn(`[Brainy] OS limit check: ${warning}`)
|
||||
}
|
||||
} catch {
|
||||
// The check must never affect open — measurement-only.
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue