feat(8.0): wire the two cor-confirmed metadata-provider contract additions
Both shapes confirmed with cor for the lockstep (cor builds the native side
against these; the JS index is a no-op / unchanged):
- probeConsistency?(): Promise<boolean> — OPTIONAL O(1) cold-open consistency
sampler on MetadataIndexProvider. brainy calls it ONCE per brain on the first
read; on `false` it self-heals via detectAndRepairCorruption() — the metadata
counterpart of the 7.33.2 graph cold-load guard, completing the phantom triad's
detect-and-repair-on-open. Best-effort: a probe failure never breaks a read
(the one-shot guard resets so a transient failure retries). The JS index omits
the method, so it is simply never probed.
- getIdsForFilter(filter, opts?: { limit?; offset? }) — brainy passes a page
bound on the UNSORTED find({ type, where, limit }) path so a native provider can
early-stop and return only the [0, limit) prefix, killing the O(N) FFI marshal
at billion scale (pairs with cor #75). brainy passes offset:0 and ALWAYS
re-windows the result itself (visibility filter + slice); the JS index ignores
opts and returns all matches (behaviour unchanged).
New unit tests cover brainy's call behaviour for both (probe→repair on false,
healthy→no-repair, failure-is-best-effort, once-per-brain; opts passed with
offset 0 on the unsorted path). Full gate green: unit 1718, integration 607.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
3f9f140c8c
commit
8b191224ce
4 changed files with 168 additions and 5 deletions
|
|
@ -401,6 +401,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* during rebuild is NOT recorded here — it re-throws and aborts init() loudly.
|
||||
*/
|
||||
private _indexRebuildFailed: Error | null = null
|
||||
/** One-shot guard so the metadata cold-open consistency probe runs once per brain. */
|
||||
private _metadataConsistencyProbed = false
|
||||
|
||||
// Sub-APIs (lazy-loaded)
|
||||
private _nlp?: NaturalLanguageProcessor
|
||||
|
|
@ -4729,6 +4731,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// This is a production-safe, concurrency-controlled lazy load
|
||||
await this.ensureIndexesLoaded()
|
||||
|
||||
// One-shot cold-open self-heal: an O(1) probe of the metadata index (when the
|
||||
// provider offers one) repairs an already-poisoned index on first read — the
|
||||
// metadata counterpart of the graph cold-load guard. No-op for the JS index.
|
||||
await this.ensureMetadataConsistencyProbed()
|
||||
|
||||
// Parse natural language queries
|
||||
let params: FindParams<T> =
|
||||
typeof query === 'string' ? await this.parseNaturalQuery(query) : query
|
||||
|
|
@ -4838,8 +4845,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
sortTopK
|
||||
)
|
||||
} else {
|
||||
// Just filter without sorting
|
||||
filteredIds = await this.metadataIndex.getIdsForFilter(filter)
|
||||
// Just filter without sorting. Pass a page bound so a native provider can
|
||||
// early-stop and return only the page-prefix (killing the O(N) FFI marshal
|
||||
// at billion scale). Over-fetch by the hidden count, then re-window below;
|
||||
// offset stays 0 because the visibility filter + slice happen here. The JS
|
||||
// index ignores the bound and returns all matches (behaviour unchanged).
|
||||
const pageEnd = (params.offset || 0) + (params.limit || 10) + hiddenIds.size
|
||||
filteredIds = await this.metadataIndex.getIdsForFilter(filter, { limit: pageEnd, offset: 0 })
|
||||
}
|
||||
|
||||
// Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact.
|
||||
|
|
@ -12913,6 +12925,42 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the optional metadata cold-open consistency probe at most once per brain.
|
||||
* When the active provider exposes `probeConsistency()` (the native cross-bucket
|
||||
* O(1) sampler), a `false` result triggers `detectAndRepairCorruption()` so an
|
||||
* already-poisoned index self-heals on first read — the metadata counterpart of
|
||||
* the 7.33.2 graph cold-load guard. Best-effort: a probe failure never breaks the
|
||||
* read (the guard is reset so a transient failure retries). No-op for the JS index
|
||||
* (it exposes no probe), and the full-scan `validateConsistency` stays the explicit
|
||||
* deep diagnostic via `validateIndexConsistency()`.
|
||||
*/
|
||||
private async ensureMetadataConsistencyProbed(): Promise<void> {
|
||||
if (this._metadataConsistencyProbed) return
|
||||
this._metadataConsistencyProbed = true
|
||||
const provider = this.metadataIndex as {
|
||||
probeConsistency?: () => Promise<boolean>
|
||||
detectAndRepairCorruption?: () => Promise<void>
|
||||
}
|
||||
if (typeof provider.probeConsistency !== 'function') return
|
||||
try {
|
||||
const healthy = await provider.probeConsistency()
|
||||
if (!healthy && typeof provider.detectAndRepairCorruption === 'function') {
|
||||
if (!this.config.silent) {
|
||||
console.warn('[Brainy] metadata index failed the cold-open consistency probe — self-healing via rebuild.')
|
||||
}
|
||||
await provider.detectAndRepairCorruption()
|
||||
}
|
||||
} catch (error) {
|
||||
// The self-heal is best-effort and must never break a read. Reset the guard
|
||||
// so a transient probe failure is retried on the next read.
|
||||
this._metadataConsistencyProbed = false
|
||||
if (!this.config.silent) {
|
||||
console.warn('[Brainy] metadata cold-open consistency probe failed (continuing):', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect and repair corrupted metadata indexes
|
||||
*
|
||||
|
|
|
|||
|
|
@ -129,7 +129,18 @@ export interface MetadataIndexProvider {
|
|||
removeFromIndex(id: string, metadata?: any): Promise<void>
|
||||
|
||||
getIds(field: string, value: any): Promise<string[]>
|
||||
getIdsForFilter(filter: any): Promise<string[]>
|
||||
/**
|
||||
* Resolve a `where` filter to its matching ids.
|
||||
* @param filter - The filter shape (eq / allOf / anyOf / ne / range / exists / …).
|
||||
* @param opts - OPTIONAL page bound for the UNSORTED `find({ type, where, limit })`
|
||||
* path. Brainy passes `{ limit: offset+limit (+hidden over-fetch), offset: 0 }` and
|
||||
* ALWAYS re-windows the result itself (visibility filter + `slice`), so a provider
|
||||
* that honors `opts` should early-stop and return the `[0, limit)` PREFIX (it must
|
||||
* NOT pre-apply `offset`). The JS index ignores `opts` and returns all matches —
|
||||
* so honoring it is a pure native-side optimization that removes the O(N) FFI
|
||||
* marshal at billion scale. Pairs with {@link getIdSetForFilter}.
|
||||
*/
|
||||
getIdsForFilter(filter: any, opts?: { limit?: number; offset?: number }): Promise<string[]>
|
||||
/**
|
||||
* @description OPTIONAL: resolve a `where` filter to its matching id universe as
|
||||
* an {@link OpaqueIdSet} (a serialized roaring `Buffer`) WITHOUT materializing
|
||||
|
|
@ -171,6 +182,17 @@ export interface MetadataIndexProvider {
|
|||
getAllVFSEntityCounts(): Promise<Map<string, number>>
|
||||
|
||||
detectAndRepairCorruption(): Promise<void>
|
||||
/**
|
||||
* OPTIONAL cheap (O(1)) cold-open consistency probe — the counterpart of the
|
||||
* graph cold-load guard. Returns `true` if a sampled `(field, value, int)` is
|
||||
* clean, `false` if it detects the cross-bucket phantom signature (an int whose
|
||||
* current value for `field` no longer equals `value`). Brainy calls it ONCE per
|
||||
* brain on the first read and, on `false`, runs {@link detectAndRepairCorruption}
|
||||
* to self-heal — so an already-poisoned index repairs itself on open without the
|
||||
* cost of the full-scan {@link validateConsistency}. A provider that omits it is
|
||||
* simply never auto-probed (no behavior change).
|
||||
*/
|
||||
probeConsistency?(): Promise<boolean>
|
||||
validateConsistency(): Promise<{
|
||||
healthy: boolean
|
||||
avgEntriesPerEntity: number
|
||||
|
|
|
|||
|
|
@ -1788,9 +1788,12 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get IDs matching Brainy Field Operator metadata filter using indexes where possible
|
||||
* Get IDs matching a Brainy Field Operator metadata filter using indexes where possible.
|
||||
* The optional `_opts` page bound is part of the provider contract for the native
|
||||
* index (early-stop at `offset+limit`); the JS index returns ALL matches and lets
|
||||
* the caller window them, so `_opts` is intentionally ignored here.
|
||||
*/
|
||||
async getIdsForFilter(filter: any): Promise<string[]> {
|
||||
async getIdsForFilter(filter: any, _opts?: { limit?: number; offset?: number }): Promise<string[]> {
|
||||
if (!filter || Object.keys(filter).length === 0) {
|
||||
return []
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue