fix: no script shape can hang on brainy's internals — unref every maintenance timer + one-shot beforeExit
A consumer's clean-room verification of the 8.0.10 fix found relate() still
hanging their scripts. Root-causing the CLASS instead of the repro found two
mechanisms:
1. The 'beforeExit' auto-flush hook looped forever on any script that never
reaches close(): Node re-emits beforeExit after every event-loop drain, and
the async flush schedules new work — flush, drain, flush, forever. The
listener now self-deregisters BEFORE its one flush, so the next drain exits.
(Empirically: process.on('beforeExit', async () => await anything) alone
never exits — this was the deepest root of the whole hang class.)
2. Every background-maintenance interval is now unref'd at creation — graph
auto-flush, LSM compaction, metadata write-buffer flush, VFS cache
maintenance, PathResolver cache maintenance, statistics debounce (the
writer-lock heartbeat, flush watcher, and cache monitors already were).
Durability is owned by close() and the beforeExit flush, both deterministic;
a best-effort interval must never keep the host process alive.
Proof: the reported shape (add x2 + relate, filesystem) exits cleanly BOTH
with close() (~0.5 s) and with NO teardown at all — and in the no-teardown
case the beforeExit flush still lands the data (verified by reopen: both
nouns + the edge present). New per-op-class sweep test asserts no ref'd
timer survives close() for add / relate / graph find / metadata update /
vfs — turning this bug class off permanently instead of per-repro.
This commit is contained in:
parent
2da2736ac6
commit
30eacbdfeb
8 changed files with 158 additions and 0 deletions
|
|
@ -1262,6 +1262,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
process.exit(0)
|
process.exit(0)
|
||||||
}
|
}
|
||||||
Brainy.beforeExitListener = async () => {
|
Brainy.beforeExitListener = async () => {
|
||||||
|
// Self-deregister FIRST: Node re-emits 'beforeExit' after every event-
|
||||||
|
// loop drain, and this flush schedules new async work — with the
|
||||||
|
// listener still attached, a script that never calls close() would spin
|
||||||
|
// flush → drain → flush forever and never exit. One flush, then the
|
||||||
|
// next drain finds no listener and the process exits.
|
||||||
|
if (Brainy.beforeExitListener) {
|
||||||
|
process.off('beforeExit', Brainy.beforeExitListener)
|
||||||
|
Brainy.beforeExitListener = undefined
|
||||||
|
}
|
||||||
await flushOnShutdown()
|
await flushOnShutdown()
|
||||||
}
|
}
|
||||||
process.on('SIGTERM', Brainy.sigtermListener)
|
process.on('SIGTERM', Brainy.sigtermListener)
|
||||||
|
|
|
||||||
|
|
@ -878,6 +878,11 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
||||||
this.flushTimer = setInterval(async () => {
|
this.flushTimer = setInterval(async () => {
|
||||||
await this.flush()
|
await this.flush()
|
||||||
}, this.config.flushInterval)
|
}, this.config.flushInterval)
|
||||||
|
// Background maintenance must never keep the host process alive —
|
||||||
|
// close()/flush() handle durability; the interval is best-effort.
|
||||||
|
if (typeof this.flushTimer.unref === 'function') {
|
||||||
|
this.flushTimer.unref()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -517,6 +517,11 @@ export class LSMTree {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, this.config.compactionInterval)
|
}, this.config.compactionInterval)
|
||||||
|
// Background compaction must never keep the host process alive —
|
||||||
|
// close() compacts/flushes deterministically; this interval is best-effort.
|
||||||
|
if (this.compactionTimer && typeof this.compactionTimer.unref === 'function') {
|
||||||
|
this.compactionTimer.unref()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -431,6 +431,12 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
this.statisticsBatchUpdateTimerId = setTimeout(() => {
|
this.statisticsBatchUpdateTimerId = setTimeout(() => {
|
||||||
this.flushStatistics()
|
this.flushStatistics()
|
||||||
}, delayMs)
|
}, delayMs)
|
||||||
|
// Best-effort statistics flush — must not keep the process alive
|
||||||
|
// (close() flushes counts deterministically).
|
||||||
|
const statsTimer = this.statisticsBatchUpdateTimerId as unknown as { unref?: () => void }
|
||||||
|
if (statsTimer && typeof statsTimer.unref === 'function') {
|
||||||
|
statsTimer.unref()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -104,6 +104,11 @@ export class MetadataWriteBuffer {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}, this.flushIntervalMs)
|
}, this.flushIntervalMs)
|
||||||
|
// Best-effort background flush — must not keep the process alive
|
||||||
|
// (close() drains the buffer for durability).
|
||||||
|
if (this.flushTimer && typeof this.flushTimer.unref === 'function') {
|
||||||
|
this.flushTimer.unref()
|
||||||
|
}
|
||||||
|
|
||||||
// Prevent timer from keeping the process alive
|
// Prevent timer from keeping the process alive
|
||||||
if (this.flushTimer && typeof this.flushTimer === 'object' && 'unref' in this.flushTimer) {
|
if (this.flushTimer && typeof this.flushTimer === 'object' && 'unref' in this.flushTimer) {
|
||||||
|
|
|
||||||
|
|
@ -525,6 +525,10 @@ export class PathResolver {
|
||||||
console.log(`[PathResolver] Cache stats: ${Math.round(hitRate * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`)
|
console.log(`[PathResolver] Cache stats: ${Math.round(hitRate * 100)}% hit rate, ${this.pathCache.size} entries, ${this.hotPaths.size} hot paths`)
|
||||||
}
|
}
|
||||||
}, 60000) // Every minute
|
}, 60000) // Every minute
|
||||||
|
// Cache maintenance must never keep the host process alive.
|
||||||
|
if (this.maintenanceTimer && typeof this.maintenanceTimer.unref === 'function') {
|
||||||
|
this.maintenanceTimer.unref()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -1546,6 +1546,10 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, 60000) // Every minute
|
}, 60000) // Every minute
|
||||||
|
// Cache maintenance must never keep the host process alive.
|
||||||
|
if (this.backgroundTimer && typeof this.backgroundTimer.unref === 'function') {
|
||||||
|
this.backgroundTimer.unref()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private getDefaultConfig(): Required<Omit<VFSConfig, 'rootEntityId'>> & { rootEntityId?: string } {
|
private getDefaultConfig(): Required<Omit<VFSConfig, 'rootEntityId'>> & { rootEntityId?: string } {
|
||||||
|
|
|
||||||
120
tests/unit/process-exit-sweep.test.ts
Normal file
120
tests/unit/process-exit-sweep.test.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
||||||
|
/**
|
||||||
|
* Process-exit sweep — no operation class may leave a ref'd timer behind.
|
||||||
|
*
|
||||||
|
* A consumer's clean-room verification of the close()-hang fix found the
|
||||||
|
* add-only path clean but `relate()` leaving ~5 uncleared Timeouts — the fix
|
||||||
|
* had covered a repro, not the bug class. This sweep turns the class off:
|
||||||
|
* for EVERY op family (add / relate / graph find / vfs), assert that after
|
||||||
|
* `close()` no NEW ref'd Timeout survives (unref'd timers are fine — they
|
||||||
|
* never keep the host process alive; that is the standard for every
|
||||||
|
* background-maintenance interval in brainy).
|
||||||
|
*
|
||||||
|
* Filesystem storage on purpose: it is the reported environment and the one
|
||||||
|
* that arms the most timers (LSM flush/compaction, write buffers, watchers).
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, afterEach } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { Brainy, NounType, VerbType } from '../../src/index.js'
|
||||||
|
|
||||||
|
const tmpDirs: string[] = []
|
||||||
|
function mkTmp(): string {
|
||||||
|
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-exit-sweep-'))
|
||||||
|
tmpDirs.push(d)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
afterEach(() => {
|
||||||
|
for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
const V = () => Array.from({ length: 384 }, () => Math.random())
|
||||||
|
|
||||||
|
/** Identity set of currently-ref'd Timeout handles (the only kind that can
|
||||||
|
* keep the event loop — and a bare consumer script — alive). */
|
||||||
|
function refdTimeouts(): Set<unknown> {
|
||||||
|
const handles: any[] = (process as any)._getActiveHandles?.() ?? []
|
||||||
|
return new Set(
|
||||||
|
handles.filter(
|
||||||
|
(h) =>
|
||||||
|
h?.constructor?.name === 'Timeout' &&
|
||||||
|
(typeof h.hasRef !== 'function' || h.hasRef())
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Describe a leaked timer well enough to find its source. */
|
||||||
|
function describeTimer(h: any): string {
|
||||||
|
const fn = h?._onTimeout
|
||||||
|
return `Timeout(after=${h?._idleTimeout}ms, repeat=${!!h?._repeat}, fn=${
|
||||||
|
fn?.name || 'anon'
|
||||||
|
}: ${String(fn).slice(0, 120).replace(/\s+/g, ' ')})`
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sweep(run: (brain: any) => Promise<void>): Promise<string[]> {
|
||||||
|
const before = refdTimeouts()
|
||||||
|
const brain: any = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: mkTmp() },
|
||||||
|
plugins: [],
|
||||||
|
silent: true
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
await run(brain)
|
||||||
|
await brain.close()
|
||||||
|
return [...refdTimeouts()].filter((h) => !before.has(h)).map(describeTimer)
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('Process-exit sweep — close() leaves no ref’d timers, per op class', () => {
|
||||||
|
it('add', async () => {
|
||||||
|
const leaked = await sweep(async (brain) => {
|
||||||
|
await brain.add({ data: 'solo', type: NounType.Concept, subtype: 's', vector: V() })
|
||||||
|
})
|
||||||
|
expect(leaked).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('add + relate (the reported hang shape)', async () => {
|
||||||
|
const leaked = await sweep(async (brain) => {
|
||||||
|
const a = await brain.add({ data: 'a', type: NounType.Concept, subtype: 's', vector: V() })
|
||||||
|
const b = await brain.add({ data: 'b', type: NounType.Concept, subtype: 's', vector: V() })
|
||||||
|
await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 's' })
|
||||||
|
})
|
||||||
|
expect(leaked).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('relate + graph find + related()', async () => {
|
||||||
|
const leaked = await sweep(async (brain) => {
|
||||||
|
const ids: string[] = []
|
||||||
|
for (let i = 0; i < 6; i++) {
|
||||||
|
ids.push(await brain.add({ data: `n${i}`, type: NounType.Concept, subtype: 's', vector: V() }))
|
||||||
|
}
|
||||||
|
for (let i = 0; i + 1 < ids.length; i++) {
|
||||||
|
await brain.relate({ from: ids[i], to: ids[i + 1], type: VerbType.RelatedTo, subtype: 's' })
|
||||||
|
}
|
||||||
|
await brain.find({ connected: { from: ids[0], depth: 2 } })
|
||||||
|
await brain.related({ node: ids[1] })
|
||||||
|
})
|
||||||
|
expect(leaked).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('metadata find + update (write-buffer path)', async () => {
|
||||||
|
const leaked = await sweep(async (brain) => {
|
||||||
|
const id = await brain.add({
|
||||||
|
data: 'meta', type: NounType.Concept, subtype: 's',
|
||||||
|
metadata: { wave: 1 }, vector: V()
|
||||||
|
})
|
||||||
|
await brain.update({ id, metadata: { wave: 2 } })
|
||||||
|
await brain.find({ where: { wave: { greaterThan: 1 } } })
|
||||||
|
})
|
||||||
|
expect(leaked).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('vfs write/read/tree (VFS + PathResolver timers)', async () => {
|
||||||
|
const leaked = await sweep(async (brain) => {
|
||||||
|
await brain.vfs.writeFile('/notes/a.md', 'alpha')
|
||||||
|
await brain.vfs.readFile('/notes/a.md')
|
||||||
|
await brain.vfs.getTreeStructure('/', { maxDepth: 2 })
|
||||||
|
})
|
||||||
|
expect(leaked).toEqual([])
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue