fix: full-removal canonical deletes + family-scoped migration gate
Two production-reported spine fixes: - Canonical delete is a FULL removal: remove()/removeMany() deleted the metadata leg but never the canonical vectors.json leg or the <id>/ directory, leaving ghost rows that inflated enumerated counts forever, read as damage scars, and caused duplicate readdir entries for re-created VFS paths (the stale-unpost path). The delete operation now routes through storage.deleteNoun — both legs + the entity container — with a full two-leg before-image rollback; deleteVerb symmetric; the blind 'no metadata file' catch that masked real faults is gone. The generation log still holds the delete's before-image, so asOf() history is unchanged. repairIndex() gains a conservative, loud orphan-prune sweep (containers with no metadata content leg) + count recompute for stores damaged by earlier versions. - Family-scoped migration gate: every read blocked on the whole-brain migration lock even when the migrating index was irrelevant. The gate now scopes to the index families a read actually consults — canonical reads never wait; find() waits only on its query shape's families; graph traversals wait only on the graph family. Writes keep the conservative whole-brain wait. A read that needs the migrating family still blocks (bounded) with the retryable MigrationInProgressError.
This commit is contained in:
parent
1d26988963
commit
366f9a91f5
8 changed files with 657 additions and 66 deletions
94
tests/unit/brainy/migration-gate-family-scoped.test.ts
Normal file
94
tests/unit/brainy/migration-gate-family-scoped.test.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
/**
|
||||
* @module tests/unit/brainy/migration-gate-family-scoped
|
||||
* @description The coordinated migration LOCK is FAMILY-SCOPED: a read waits only
|
||||
* on the index families it actually consults. A one-time native migration of ONE
|
||||
* family (its `isMigrating()` held) must NOT block a read served entirely from
|
||||
* canonical storage (get / VFS content + dir) or from a HEALTHY family — only a
|
||||
* read that needs the migrating family blocks. Regression for the whole-brain
|
||||
* gate that hung getStats / readdir / readFile behind an unrelated family's
|
||||
* migration until the wait timed out.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { Brainy } from '../../../src/brainy.js'
|
||||
import { MigrationInProgressError } from '../../../src/errors/brainyError.js'
|
||||
|
||||
const seed = async () => {
|
||||
const brain = new Brainy({
|
||||
storage: { type: 'memory' },
|
||||
dimensions: 384,
|
||||
requireSubtype: false,
|
||||
silent: true,
|
||||
// Short so a genuine block resolves fast; a real block would otherwise wait
|
||||
// the 30 s default — this test asserts the wait does NOT happen for scoped
|
||||
// reads, and DOES (bounded) for the one that needs the migrating family.
|
||||
migrationWaitTimeoutMs: 300
|
||||
})
|
||||
await brain.init()
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } })
|
||||
}
|
||||
await brain.vfs.writeFile('/notes/hello.txt', 'canonical bytes — no index needed')
|
||||
await brain.flush()
|
||||
return brain
|
||||
}
|
||||
|
||||
/** Force a provider to report an in-flight (stuck) migration. */
|
||||
const jam = (provider: unknown) => {
|
||||
;(provider as { isMigrating: () => boolean }).isMigrating = () => true
|
||||
}
|
||||
|
||||
describe('migration LOCK is family-scoped', () => {
|
||||
beforeEach(() => {
|
||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||
})
|
||||
|
||||
it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => {
|
||||
const brain = await seed()
|
||||
const childId = (
|
||||
(await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }>
|
||||
)[0].entityId
|
||||
|
||||
jam((brain as any).index) // vector provider migrating; graph + metadata healthy
|
||||
|
||||
// None of these consult the vector family → they serve immediately.
|
||||
await expect(brain.getStats()).resolves.toBeDefined()
|
||||
await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) // graph traversal
|
||||
await expect(brain.vfs.readFile('/notes/hello.txt')).resolves.toBeDefined() // canonical blob
|
||||
await expect(brain.get(childId)).resolves.not.toBeNull() // canonical by id
|
||||
await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined() // metadata filter
|
||||
})
|
||||
|
||||
it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => {
|
||||
const brain = await seed()
|
||||
jam((brain as any).index)
|
||||
|
||||
// A semantic query consults the vector index — it must wait, and (bounded by
|
||||
// migrationWaitTimeoutMs) surface the retryable MigrationInProgressError
|
||||
// rather than serve a half-built result.
|
||||
await expect(brain.find({ query: 'doc' })).rejects.toBeInstanceOf(MigrationInProgressError)
|
||||
})
|
||||
|
||||
it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => {
|
||||
const brain = await seed()
|
||||
const childId = (
|
||||
(await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }>
|
||||
)[0].entityId
|
||||
|
||||
jam((brain as any).graphIndex) // graph provider migrating; vector + metadata healthy
|
||||
|
||||
// Canonical + vector + metadata are unaffected.
|
||||
await expect(brain.get(childId)).resolves.not.toBeNull()
|
||||
await expect(brain.find({ query: 'doc' })).resolves.toBeDefined()
|
||||
await expect(brain.find({ where: { i: 1 } })).resolves.toBeDefined()
|
||||
|
||||
// A graph traversal needs the migrating family → it blocks (bounded).
|
||||
await expect(brain.related({ from: childId })).rejects.toBeInstanceOf(MigrationInProgressError)
|
||||
})
|
||||
|
||||
it('with no migration in flight, every read serves (the fast path is a no-op)', async () => {
|
||||
const brain = await seed()
|
||||
await expect(brain.getStats()).resolves.toBeDefined()
|
||||
await expect(brain.find({ query: 'doc' })).resolves.toBeDefined()
|
||||
await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
|
@ -2,11 +2,16 @@
|
|||
* Migration LOCK (#18) — coordinated, automatic 7.x → 8.0 auto-upgrade.
|
||||
*
|
||||
* Exercises the real block-and-queue behavior of `awaitMigrationLock` at the
|
||||
* `ensureInitialized` choke point: while ANY index provider reports
|
||||
* `isMigrating() === true`, data-plane reads and writes WAIT (no operation
|
||||
* touches a half-built index); observability (`getIndexStatus`, `health`) and
|
||||
* the lock-clearing `stampBrainFormat` stay exempt; and a lock that outlives the
|
||||
* configured window surfaces a retryable `MigrationInProgressError`.
|
||||
* `ensureInitialized` choke point. The gate is FAMILY-SCOPED: a write (which
|
||||
* touches every index) WAITS while ANY provider reports `isMigrating() === true`,
|
||||
* and a read WAITS only when the migrating provider is one of the index families
|
||||
* that read consults (so a read served from canonical storage or a healthy family
|
||||
* is never blocked by an unrelated family's migration — see
|
||||
* `tests/unit/brainy/migration-gate-family-scoped.test.ts`). Observability
|
||||
* (`getIndexStatus`, `health`) and the lock-clearing `stampBrainFormat` stay
|
||||
* exempt; a lock that outlives the configured window surfaces a retryable
|
||||
* `MigrationInProgressError`. Here the graph provider holds the lock, so the
|
||||
* read cases use a graph traversal (`related`) — the family that actually waits.
|
||||
*
|
||||
* A native (cortex) provider owns the real migration; here we simulate the lock
|
||||
* by feature-detect-injecting `isMigrating()` onto a live provider, exactly as
|
||||
|
|
@ -84,11 +89,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => {
|
|||
expect(id).toBeTruthy()
|
||||
})
|
||||
|
||||
it('blocks a read while migrating, then releases when the flag clears (poll path)', async () => {
|
||||
it('blocks a graph read while the graph provider migrates, then releases when the flag clears (poll path)', async () => {
|
||||
const anchor = await brain.add({ data: 'anchor', type: NounType.Concept })
|
||||
let migrating = true
|
||||
brain.graphIndex.isMigrating = () => migrating
|
||||
|
||||
const p = brain.find({ type: NounType.Concept }) // a read → gated
|
||||
const p = brain.related({ from: anchor }) // a GRAPH read → gated on the graph migration
|
||||
let resolved = false
|
||||
p.then(() => {
|
||||
resolved = true
|
||||
|
|
@ -109,9 +115,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => {
|
|||
migrationWaitTimeoutMs: 120
|
||||
})
|
||||
await shortBrain.init()
|
||||
setMigrating(shortBrain, true) // never clears
|
||||
const anchor = await shortBrain.add({ data: 'anchor', type: NounType.Concept })
|
||||
setMigrating(shortBrain, true) // graph lock never clears
|
||||
|
||||
await expect(shortBrain.find({ type: NounType.Concept })).rejects.toBeInstanceOf(
|
||||
// A graph read needs the migrating family → it waits out the window and
|
||||
// surfaces the retryable error rather than serving a half-built traversal.
|
||||
await expect(shortBrain.related({ from: anchor })).rejects.toBeInstanceOf(
|
||||
MigrationInProgressError
|
||||
)
|
||||
try {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue