fix(open): a provider rebuilding itself is a third state, not a CRITICAL
Follower to the self-rebuild deference. The open gate's consistency check — "metadata index has 0 entries but storage has N entities" → CRITICAL + a forced second rebuild — knew two states, migrating and not. A provider whose rebuild() returns once the rebuild is OWNED AND RUNNING ONLINE (its doors refusing by name while other families serve) legitimately reports 0 entries there, so every first contact printed a false CRITICAL and kicked a redundant second rebuild. The exemption rides the rebuild-progress hook, NOT isMigrating() — widening that would hold every write and 503 the whole brain through the migration snapshot, which is worse than the false alarm. The check's real class is untouched: a provider reporting 0 entries with no rebuild in progress still trips it. The crash-recovery rebuild kick gets the same deference: a provider already rebuilding itself from canonical is doing exactly that work, and the fold ran in the generation store's open before any provider existed, so what it is reading is the repaired canonical. Pin: a provider stub reporting a rebuild and 0 entries opens with no CRITICAL line and no second rebuild; the vacuous-stub case fails loudly.
This commit is contained in:
parent
131daa08cd
commit
50676c02f4
2 changed files with 76 additions and 3 deletions
|
|
@ -1489,10 +1489,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
`[Brainy] Rebuilding indexes after crash recovery rolled back ` +
|
`[Brainy] Rebuilding indexes after crash recovery rolled back ` +
|
||||||
`${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)`
|
`${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)`
|
||||||
)
|
)
|
||||||
|
// SELF-REBUILD DEFERENCE, same law as the open gate: a provider that
|
||||||
|
// is already rebuilding itself from canonical is doing exactly this
|
||||||
|
// work. Kicking a second rebuild on top of it is redundant at best.
|
||||||
|
// Safe by ordering: the crash-recovery fold ran in the generation
|
||||||
|
// store's open, BEFORE any provider was constructed, so a provider
|
||||||
|
// rebuilding now is reading the repaired canonical records.
|
||||||
|
const kick = async (leg: string, provider: { rebuild: () => Promise<void> }) => {
|
||||||
|
const rebuilding = assessProviderRebuild(provider)
|
||||||
|
if (rebuilding) {
|
||||||
|
prodLog.narrate(
|
||||||
|
`[Brainy] crash-recovery rebuild: the ${leg} provider is already ` +
|
||||||
|
`${describeRebuildProgress(rebuilding)} from canonical — not kicking a second one.`
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await provider.rebuild()
|
||||||
|
}
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
this.metadataIndex.rebuild(),
|
kick('metadata', this.metadataIndex),
|
||||||
this.index.rebuild(),
|
kick('vector', this.index as unknown as { rebuild: () => Promise<void> }),
|
||||||
this.graphIndex.rebuild()
|
kick('graph', this.graphIndex)
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -17757,6 +17774,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// when the metadata provider holds the migration lock: a 0 count there
|
// when the metadata provider holds the migration lock: a 0 count there
|
||||||
// reflects its in-place rebuild in progress, not a missed rebuild, so
|
// reflects its in-place rebuild in progress, not a missed rebuild, so
|
||||||
// forcing a second rebuild would collide with the provider's own.
|
// forcing a second rebuild would collide with the provider's own.
|
||||||
|
// THREE states, not two. `metadataMigrating` above is true for a
|
||||||
|
// provider holding the migration lock AND for one that reports it is
|
||||||
|
// rebuilding itself — a provider whose rebuild() returns once the
|
||||||
|
// rebuild is OWNED AND RUNNING (online, its doors refusing by name)
|
||||||
|
// legitimately reports 0 entries here, and calling that CRITICAL would
|
||||||
|
// print a false alarm and kick a redundant second rebuild on every
|
||||||
|
// first contact. The check's real class — a rebuild that ran to
|
||||||
|
// completion and produced nothing — is untouched: a provider reporting
|
||||||
|
// 0 entries with NO rebuild in progress still trips it.
|
||||||
if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) {
|
if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) {
|
||||||
console.error(
|
console.error(
|
||||||
`[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` +
|
`[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` +
|
||||||
|
|
|
||||||
|
|
@ -142,4 +142,51 @@ describe('a provider rebuilding itself never blocks open', () => {
|
||||||
;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false
|
;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false
|
||||||
await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined()
|
await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined()
|
||||||
}, 180_000)
|
}, 180_000)
|
||||||
|
|
||||||
|
it('a rebuilding provider reporting 0 entries is not a CRITICAL, and gets no second rebuild', async () => {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-critical-'))
|
||||||
|
dirs.push(dir)
|
||||||
|
const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
await seed.init()
|
||||||
|
await seed.add({ data: 'a stored entity', type: NounType.Concept })
|
||||||
|
await seed.flush()
|
||||||
|
await seed.close()
|
||||||
|
|
||||||
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||||
|
brains.push(brain)
|
||||||
|
|
||||||
|
let rebuildCalls = 0
|
||||||
|
const errors: string[] = []
|
||||||
|
const origError = console.error
|
||||||
|
console.error = ((...a: unknown[]) => { errors.push(a.map(String).join(' ')) }) as typeof console.error
|
||||||
|
|
||||||
|
const inner = brain as unknown as { metadataIndex: Record<string, unknown> }
|
||||||
|
const patcher = setInterval(() => {
|
||||||
|
if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) {
|
||||||
|
const target = inner.metadataIndex
|
||||||
|
target.rebuildInProgress = () => ({ phase: 'online metadata rebuild', startedAt: Date.now() })
|
||||||
|
target.healthReport = () => ({
|
||||||
|
provider: 'metadata', healthy: false, serving: false,
|
||||||
|
generation: 1, invariants: [], unledgered: []
|
||||||
|
})
|
||||||
|
// The shape the native engine now has: the index reports NOTHING while
|
||||||
|
// its rebuild runs online behind refusing doors.
|
||||||
|
target.getStats = async () => ({ totalEntries: 0 })
|
||||||
|
target.rebuild = async () => { rebuildCalls++ }
|
||||||
|
}
|
||||||
|
}, 1)
|
||||||
|
try {
|
||||||
|
await brain.init()
|
||||||
|
} finally {
|
||||||
|
clearInterval(patcher)
|
||||||
|
console.error = origError
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(
|
||||||
|
typeof inner.metadataIndex.rebuildInProgress,
|
||||||
|
'the stub provider was never installed — the test is vacuous'
|
||||||
|
).toBe('function')
|
||||||
|
expect(errors.filter((l) => /CRITICAL: Metadata index has 0 entries/.test(l))).toEqual([])
|
||||||
|
expect(rebuildCalls).toBe(0)
|
||||||
|
}, 180_000)
|
||||||
})
|
})
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue