brainy/tests/unit/migration-lock.test.ts
David Snelling 366f9a91f5 fix: full-removal canonical deletes + family-scoped migration gate
Two production-reported spine fixes:

- Canonical delete is a FULL removal: remove()/removeMany() deleted the
  metadata leg but never the canonical vectors.json leg or the <id>/
  directory, leaving ghost rows that inflated enumerated counts forever,
  read as damage scars, and caused duplicate readdir entries for
  re-created VFS paths (the stale-unpost path). The delete operation now
  routes through storage.deleteNoun — both legs + the entity container —
  with a full two-leg before-image rollback; deleteVerb symmetric; the
  blind 'no metadata file' catch that masked real faults is gone. The
  generation log still holds the delete's before-image, so asOf() history
  is unchanged. repairIndex() gains a conservative, loud orphan-prune
  sweep (containers with no metadata content leg) + count recompute for
  stores damaged by earlier versions.

- Family-scoped migration gate: every read blocked on the whole-brain
  migration lock even when the migrating index was irrelevant. The gate
  now scopes to the index families a read actually consults — canonical
  reads never wait; find() waits only on its query shape's families;
  graph traversals wait only on the graph family. Writes keep the
  conservative whole-brain wait. A read that needs the migrating family
  still blocks (bounded) with the retryable MigrationInProgressError.
2026-07-14 10:11:53 -07:00

179 lines
7.3 KiB
TypeScript

/**
* Migration LOCK (#18) — coordinated, automatic 7.x → 8.0 auto-upgrade.
*
* Exercises the real block-and-queue behavior of `awaitMigrationLock` at the
* `ensureInitialized` choke point. The gate is FAMILY-SCOPED: a write (which
* touches every index) WAITS while ANY provider reports `isMigrating() === true`,
* and a read WAITS only when the migrating provider is one of the index families
* that read consults (so a read served from canonical storage or a healthy family
* is never blocked by an unrelated family's migration — see
* `tests/unit/brainy/migration-gate-family-scoped.test.ts`). Observability
* (`getIndexStatus`, `health`) and the lock-clearing `stampBrainFormat` stay
* exempt; a lock that outlives the configured window surfaces a retryable
* `MigrationInProgressError`. Here the graph provider holds the lock, so the
* read cases use a graph traversal (`related`) — the family that actually waits.
*
* A native (cortex) provider owns the real migration; here we simulate the lock
* by feature-detect-injecting `isMigrating()` onto a live provider, exactly as
* the production feature-detection reads it.
*/
import { describe, it, expect, beforeEach } from 'vitest'
import { Brainy, NounType, MigrationInProgressError } from '../../src/index.js'
import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js'
/** Force the graph provider to hold (or release) the migration lock. */
function setMigrating(brain: any, migrating: boolean): void {
brain.graphIndex.isMigrating = () => migrating
}
describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => {
let brain: any
beforeEach(async () => {
brain = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
migrationWaitTimeoutMs: 5000 // generous; timing tests clear well within it
})
await brain.init()
})
it('does not gate operations when no provider is migrating (fast path)', async () => {
const id = await brain.add({ data: 'hello', type: NounType.Concept })
expect(id).toBeTruthy()
const status = await brain.getIndexStatus()
expect(status.migrating).toBe(false)
expect(status.migration).toBeUndefined()
})
it('getIndexStatus() reports migrating WITHOUT blocking (lock-exempt observability)', async () => {
setMigrating(brain, true)
const status = await brain.getIndexStatus() // must resolve, never hang
expect(status.migrating).toBe(true)
expect(status.migration).toBeDefined()
expect(typeof status.migration.elapsedMs).toBe('number')
})
it('health() surfaces the migration as a warn check without blocking', async () => {
setMigrating(brain, true)
const h = await brain.health() // lock-exempt
expect(h.overall).toBe('warn')
expect(h.checks.some((c: any) => c.name === 'migration')).toBe(true)
})
it('checkHealth() answers during a migration (lock-exempt diagnostics)', async () => {
setMigrating(brain, true)
const r = await brain.checkHealth()
expect(r).toHaveProperty('healthy')
})
it('blocks a write while migrating (poll path), then completes when the lock clears', async () => {
setMigrating(brain, true)
let resolved = false
const p = brain
.add({ data: 'queued write', type: NounType.Concept })
.then((id: string) => {
resolved = true
return id
})
// Still blocked shortly after issue.
await new Promise((r) => setTimeout(r, 40))
expect(resolved).toBe(false)
// Clearing the lock releases the queued write against the good indexes.
setMigrating(brain, false)
const id = await p
expect(resolved).toBe(true)
expect(id).toBeTruthy()
})
it('blocks a graph read while the graph provider migrates, then releases when the flag clears (poll path)', async () => {
const anchor = await brain.add({ data: 'anchor', type: NounType.Concept })
let migrating = true
brain.graphIndex.isMigrating = () => migrating
const p = brain.related({ from: anchor }) // a GRAPH read → gated on the graph migration
let resolved = false
p.then(() => {
resolved = true
})
await new Promise((r) => setTimeout(r, 40))
expect(resolved).toBe(false)
// Clearing the flag lets the next poll release the read against good indexes.
migrating = false
await expect(p).resolves.toEqual([])
})
it('throws a retryable MigrationInProgressError when the lock outlives the timeout', async () => {
const shortBrain: any = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
migrationWaitTimeoutMs: 120
})
await shortBrain.init()
const anchor = await shortBrain.add({ data: 'anchor', type: NounType.Concept })
setMigrating(shortBrain, true) // graph lock never clears
// A graph read needs the migrating family → it waits out the window and
// surfaces the retryable error rather than serving a half-built traversal.
await expect(shortBrain.related({ from: anchor })).rejects.toBeInstanceOf(
MigrationInProgressError
)
try {
await shortBrain.add({ data: 'x', type: NounType.Concept })
throw new Error('expected MigrationInProgressError')
} catch (e: any) {
expect(e).toBeInstanceOf(MigrationInProgressError)
expect(e.retryable).toBe(true)
expect(typeof e.elapsedMs).toBe('number')
}
})
it('stampBrainFormat() is NOT gated — cor clears the lock through it (no deadlock)', async () => {
setMigrating(brain, true)
// Must resolve, not hang, even while a migration is "in progress".
await expect(brain.stampBrainFormat()).resolves.toBeUndefined()
})
it('close() is NOT gated — an instance can shut down mid-migration', async () => {
setMigrating(brain, true)
await expect(brain.close()).resolves.toBeUndefined()
})
it('C1: init() waits out a migration BEFORE VFS bootstrap (no deadlock, no failure)', async () => {
// The critical regression: init()'s own VFS bootstrap (get/add/find on the
// root) routes through the gated ensureInitialized. If a provider migrates
// during init, init must WAIT for the lock to clear BEFORE bootstrapping VFS —
// otherwise it deadlocks (< timeout) or throws MigrationInProgressError past
// the timeout. We make the graph provider report migrating for a short window
// spanning init, then clear it, and assert init completes and the brain works.
const orig = (GraphAdjacencyIndex.prototype as any).isMigrating
let migrating = true
;(GraphAdjacencyIndex.prototype as any).isMigrating = () => migrating
const clearMigration = setTimeout(() => {
migrating = false
}, 60)
try {
const b: any = new Brainy({
requireSubtype: false,
storage: { type: 'memory' },
migrationWaitTimeoutMs: 5000
})
await b.init() // must resolve once the migration clears — not hang, not throw
expect(b.isInitialized).toBe(true)
// VFS bootstrapped post-migration → normal write/read work end-to-end.
const id = await b.add({ data: 'after upgrade', type: NounType.Concept })
expect(id).toBeTruthy()
expect(await b.get(id)).not.toBeNull()
await b.close()
} finally {
clearTimeout(clearMigration)
if (orig) (GraphAdjacencyIndex.prototype as any).isMigrating = orig
else delete (GraphAdjacencyIndex.prototype as any).isMigrating
}
})
})