|
|
|
|
@ -544,6 +544,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
|
|
* and narrate forever. Reset by {@link deregisterShutdownHooksIfIdle}. */
|
|
|
|
|
private static beforeExitNarrated = false
|
|
|
|
|
|
|
|
|
|
/** True for the entire duration of ONE `closeOnShutdown()` run (the
|
|
|
|
|
* signal-path handler in {@link registerShutdownHooks}) — from before it
|
|
|
|
|
* starts closing instances until after it has decided whether to exit.
|
|
|
|
|
* THE RACE THIS CLOSES: closing the LAST live instance calls
|
|
|
|
|
* `close()` → `deregisterShutdownHooksIfIdle()` synchronously, which
|
|
|
|
|
* removes `Brainy.sigtermListener` from `process` — while `closeOnShutdown`
|
|
|
|
|
* (that very listener's OWN still-running invocation) hasn't yet reached
|
|
|
|
|
* `exitIfSoleShutdownOwner()`'s `process.exit(0)`. In that window Node has
|
|
|
|
|
* NO registered SIGTERM listener, so a second/concurrent delivery of the
|
|
|
|
|
* same signal (a raced re-send, common on a loaded host) falls through to
|
|
|
|
|
* Node's default disposition and kills the process outright — the
|
|
|
|
|
* clean-shutdown work already finished, but the process never reports the
|
|
|
|
|
* 0 it earned. `deregisterShutdownHooksIfIdle()` checks this flag and
|
|
|
|
|
* defers; `closeOnShutdown()`'s `finally` re-runs the deregistration check
|
|
|
|
|
* once it is done, so the listener never actually leaks past its use. */
|
|
|
|
|
private static shutdownSignalHandlerActive = false
|
|
|
|
|
|
|
|
|
|
/** Poll cadence (ms) for the migration LOCK when a provider exposes no
|
|
|
|
|
* event-driven `whenMigrationComplete()` signal. See {@link awaitMigrationLock}. */
|
|
|
|
|
private static readonly MIGRATION_POLL_INTERVAL_MS = 250
|
|
|
|
|
@ -2196,51 +2213,74 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
|
|
*/
|
|
|
|
|
const closeOnShutdown = async () => {
|
|
|
|
|
console.log('Shutdown signal received - flushing pending data...')
|
|
|
|
|
// DEFER ONE MACROTASK. A host application registers its own listener on
|
|
|
|
|
// the same signal, and Node runs listeners in registration order — ours
|
|
|
|
|
// is usually first, because the brain was opened before the host wired
|
|
|
|
|
// its shutdown. Yielding once lets every other listener for this signal
|
|
|
|
|
// run its synchronous prologue, so a host that calls close() gets to be
|
|
|
|
|
// the owner. It is only a courtesy, never the safety: close()'s own
|
|
|
|
|
// single-flight gate is what makes a lost race harmless.
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve))
|
|
|
|
|
// HOLD THE LISTENER FOR THE WHOLE RUN. Closing the LAST live instance
|
|
|
|
|
// below calls close() → deregisterShutdownHooksIfIdle(), which removes
|
|
|
|
|
// Brainy's own SIGTERM/SIGINT listeners from `process` — synchronously,
|
|
|
|
|
// before THIS invocation has reached exitIfSoleShutdownOwner()'s
|
|
|
|
|
// process.exit(0). Left alone, that opens a window with no registered
|
|
|
|
|
// listener for the signal at all, so a second/concurrent delivery of
|
|
|
|
|
// the same signal (a raced re-send — not rare on a loaded host) falls
|
|
|
|
|
// through to Node's default disposition and kills the process outright
|
|
|
|
|
// AFTER the clean-shutdown work already finished, reporting a signal
|
|
|
|
|
// kill instead of the 0 the shutdown earned. Setting this flag makes
|
|
|
|
|
// deregisterShutdownHooksIfIdle() defer; the `finally` below re-checks
|
|
|
|
|
// it once this run is fully done — closeOnShutdown, not a nested
|
|
|
|
|
// close(), owns exactly when the listener actually comes off.
|
|
|
|
|
Brainy.shutdownSignalHandlerActive = true
|
|
|
|
|
try {
|
|
|
|
|
// DEFER ONE MACROTASK. A host application registers its own listener
|
|
|
|
|
// on the same signal, and Node runs listeners in registration order —
|
|
|
|
|
// ours is usually first, because the brain was opened before the
|
|
|
|
|
// host wired its shutdown. Yielding once lets every other listener
|
|
|
|
|
// for this signal run its synchronous prologue, so a host that calls
|
|
|
|
|
// close() gets to be the owner. It is only a courtesy, never the
|
|
|
|
|
// safety: close()'s own single-flight gate is what makes a lost race
|
|
|
|
|
// harmless.
|
|
|
|
|
await new Promise<void>((resolve) => setImmediate(resolve))
|
|
|
|
|
|
|
|
|
|
let closedCount = 0
|
|
|
|
|
let deferredCount = 0
|
|
|
|
|
let failedCount = 0
|
|
|
|
|
// Snapshot: close() splices Brainy.instances while we iterate.
|
|
|
|
|
for (const instance of [...Brainy.instances]) {
|
|
|
|
|
if (!instance.initialized) continue
|
|
|
|
|
// SOMEONE ELSE OWNS THIS ONE. Not a flush, not a lock release, not a
|
|
|
|
|
// component close — nothing. Touching a brain whose close is running
|
|
|
|
|
// is the whole defect this handler was rewritten for.
|
|
|
|
|
if (instance.closed || instance._closeInFlight !== null) {
|
|
|
|
|
deferredCount++
|
|
|
|
|
continue
|
|
|
|
|
let closedCount = 0
|
|
|
|
|
let deferredCount = 0
|
|
|
|
|
let failedCount = 0
|
|
|
|
|
// Snapshot: close() splices Brainy.instances while we iterate.
|
|
|
|
|
for (const instance of [...Brainy.instances]) {
|
|
|
|
|
if (!instance.initialized) continue
|
|
|
|
|
// SOMEONE ELSE OWNS THIS ONE. Not a flush, not a lock release, not a
|
|
|
|
|
// component close — nothing. Touching a brain whose close is running
|
|
|
|
|
// is the whole defect this handler was rewritten for.
|
|
|
|
|
if (instance.closed || instance._closeInFlight !== null) {
|
|
|
|
|
deferredCount++
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
// Law 1: this try/catch is the isolation — the loop continues.
|
|
|
|
|
await instance.close()
|
|
|
|
|
closedCount++
|
|
|
|
|
} catch (error) {
|
|
|
|
|
failedCount++
|
|
|
|
|
console.error('Failed to close one Brainy instance on shutdown:', error)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
// Law 1: this try/catch is the isolation — the loop continues.
|
|
|
|
|
await instance.close()
|
|
|
|
|
closedCount++
|
|
|
|
|
} catch (error) {
|
|
|
|
|
failedCount++
|
|
|
|
|
console.error('Failed to close one Brainy instance on shutdown:', error)
|
|
|
|
|
if (closedCount > 0) {
|
|
|
|
|
console.log(`Flushed successfully (${closedCount} instance${closedCount > 1 ? 's' : ''})`)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (closedCount > 0) {
|
|
|
|
|
console.log(`Flushed successfully (${closedCount} instance${closedCount > 1 ? 's' : ''})`)
|
|
|
|
|
}
|
|
|
|
|
if (deferredCount > 0) {
|
|
|
|
|
console.log(
|
|
|
|
|
`${deferredCount} Brainy instance${deferredCount > 1 ? 's are' : ' is'} already ` +
|
|
|
|
|
`closing — left to the caller that owns that close.`
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
if (failedCount > 0) {
|
|
|
|
|
console.error(
|
|
|
|
|
`${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` +
|
|
|
|
|
`their writer locks were released, but their next open will run crash recovery.`
|
|
|
|
|
)
|
|
|
|
|
if (deferredCount > 0) {
|
|
|
|
|
console.log(
|
|
|
|
|
`${deferredCount} Brainy instance${deferredCount > 1 ? 's are' : ' is'} already ` +
|
|
|
|
|
`closing — left to the caller that owns that close.`
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
if (failedCount > 0) {
|
|
|
|
|
console.error(
|
|
|
|
|
`${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` +
|
|
|
|
|
`their writer locks were released, but their next open will run crash recovery.`
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
// Release the hold and run the deferred check ourselves — the last
|
|
|
|
|
// close() above may have found the flag set and skipped its own
|
|
|
|
|
// deregistration, so nobody else will do this if we don't.
|
|
|
|
|
Brainy.shutdownSignalHandlerActive = false
|
|
|
|
|
Brainy.deregisterShutdownHooksIfIdle()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -2404,9 +2444,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|
|
|
|
* script that closed every brain exits on its own — a library must never
|
|
|
|
|
* keep its host process alive. Re-initializing later re-registers them
|
|
|
|
|
* (the `shutdownHooksRegisteredGlobally` flag resets here).
|
|
|
|
|
*
|
|
|
|
|
* Deferred (not skipped — {@link closeOnShutdown}'s `finally` always
|
|
|
|
|
* re-checks) while a signal-path shutdown is actively running: that
|
|
|
|
|
* handler's OWN still-in-flight invocation is `Brainy.sigtermListener`, and
|
|
|
|
|
* removing it out from under itself — which closing the LAST instance here
|
|
|
|
|
* would otherwise do, synchronously, mid-run — would leave `process` with
|
|
|
|
|
* no listener for the signal for the remainder of that run. See
|
|
|
|
|
* {@link shutdownSignalHandlerActive}'s doc for the exact race this closes.
|
|
|
|
|
*/
|
|
|
|
|
private static deregisterShutdownHooksIfIdle(): void {
|
|
|
|
|
if (Brainy.instances.length > 0 || !Brainy.shutdownHooksRegisteredGlobally) {
|
|
|
|
|
if (Brainy.instances.length > 0 || !Brainy.shutdownHooksRegisteredGlobally || Brainy.shutdownSignalHandlerActive) {
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if (Brainy.sigtermListener) process.off('SIGTERM', Brainy.sigtermListener)
|
|
|
|
|
|