From 8b191224ce4b619394c2e447ac6455b4e5f093e7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 29 Jun 2026 12:11:00 -0700 Subject: [PATCH] feat(8.0): wire the two cor-confirmed metadata-provider contract additions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both shapes confirmed with cor for the lockstep (cor builds the native side against these; the JS index is a no-op / unchanged): - probeConsistency?(): Promise — OPTIONAL O(1) cold-open consistency sampler on MetadataIndexProvider. brainy calls it ONCE per brain on the first read; on `false` it self-heals via detectAndRepairCorruption() — the metadata counterpart of the 7.33.2 graph cold-load guard, completing the phantom triad's detect-and-repair-on-open. Best-effort: a probe failure never breaks a read (the one-shot guard resets so a transient failure retries). The JS index omits the method, so it is simply never probed. - getIdsForFilter(filter, opts?: { limit?; offset? }) — brainy passes a page bound on the UNSORTED find({ type, where, limit }) path so a native provider can early-stop and return only the [0, limit) prefix, killing the O(N) FFI marshal at billion scale (pairs with cor #75). brainy passes offset:0 and ALWAYS re-windows the result itself (visibility filter + slice); the JS index ignores opts and returns all matches (behaviour unchanged). New unit tests cover brainy's call behaviour for both (probe→repair on false, healthy→no-repair, failure-is-best-effort, once-per-brain; opts passed with offset 0 on the unsorted path). Full gate green: unit 1718, integration 607. Co-Authored-By: Claude Opus 4.8 --- src/brainy.ts | 52 ++++++++++- src/plugin.ts | 24 ++++- src/utils/metadataIndex.ts | 7 +- .../brainy/metadata-provider-contract.test.ts | 90 +++++++++++++++++++ 4 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 tests/unit/brainy/metadata-provider-contract.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 55b80d5a..3ad695c3 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -401,6 +401,8 @@ export class Brainy implements BrainyInterface { * during rebuild is NOT recorded here — it re-throws and aborts init() loudly. */ private _indexRebuildFailed: Error | null = null + /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */ + private _metadataConsistencyProbed = false // Sub-APIs (lazy-loaded) private _nlp?: NaturalLanguageProcessor @@ -4729,6 +4731,11 @@ export class Brainy implements BrainyInterface { // This is a production-safe, concurrency-controlled lazy load await this.ensureIndexesLoaded() + // One-shot cold-open self-heal: an O(1) probe of the metadata index (when the + // provider offers one) repairs an already-poisoned index on first read — the + // metadata counterpart of the graph cold-load guard. No-op for the JS index. + await this.ensureMetadataConsistencyProbed() + // Parse natural language queries let params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query @@ -4838,8 +4845,13 @@ export class Brainy implements BrainyInterface { sortTopK ) } else { - // Just filter without sorting - filteredIds = await this.metadataIndex.getIdsForFilter(filter) + // Just filter without sorting. Pass a page bound so a native provider can + // early-stop and return only the page-prefix (killing the O(N) FFI marshal + // at billion scale). Over-fetch by the hidden count, then re-window below; + // offset stays 0 because the visibility filter + slice happen here. The JS + // index ignores the bound and returns all matches (behaviour unchanged). + const pageEnd = (params.offset || 0) + (params.limit || 10) + hiddenIds.size + filteredIds = await this.metadataIndex.getIdsForFilter(filter, { limit: pageEnd, offset: 0 }) } // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact. @@ -12913,6 +12925,42 @@ export class Brainy implements BrainyInterface { return result } + /** + * Run the optional metadata cold-open consistency probe at most once per brain. + * When the active provider exposes `probeConsistency()` (the native cross-bucket + * O(1) sampler), a `false` result triggers `detectAndRepairCorruption()` so an + * already-poisoned index self-heals on first read — the metadata counterpart of + * the 7.33.2 graph cold-load guard. Best-effort: a probe failure never breaks the + * read (the guard is reset so a transient failure retries). No-op for the JS index + * (it exposes no probe), and the full-scan `validateConsistency` stays the explicit + * deep diagnostic via `validateIndexConsistency()`. + */ + private async ensureMetadataConsistencyProbed(): Promise { + if (this._metadataConsistencyProbed) return + this._metadataConsistencyProbed = true + const provider = this.metadataIndex as { + probeConsistency?: () => Promise + detectAndRepairCorruption?: () => Promise + } + if (typeof provider.probeConsistency !== 'function') return + try { + const healthy = await provider.probeConsistency() + if (!healthy && typeof provider.detectAndRepairCorruption === 'function') { + if (!this.config.silent) { + console.warn('[Brainy] metadata index failed the cold-open consistency probe — self-healing via rebuild.') + } + await provider.detectAndRepairCorruption() + } + } catch (error) { + // The self-heal is best-effort and must never break a read. Reset the guard + // so a transient probe failure is retried on the next read. + this._metadataConsistencyProbed = false + if (!this.config.silent) { + console.warn('[Brainy] metadata cold-open consistency probe failed (continuing):', error) + } + } + } + /** * Detect and repair corrupted metadata indexes * diff --git a/src/plugin.ts b/src/plugin.ts index 736d777f..d274d831 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -129,7 +129,18 @@ export interface MetadataIndexProvider { removeFromIndex(id: string, metadata?: any): Promise getIds(field: string, value: any): Promise - getIdsForFilter(filter: any): Promise + /** + * Resolve a `where` filter to its matching ids. + * @param filter - The filter shape (eq / allOf / anyOf / ne / range / exists / …). + * @param opts - OPTIONAL page bound for the UNSORTED `find({ type, where, limit })` + * path. Brainy passes `{ limit: offset+limit (+hidden over-fetch), offset: 0 }` and + * ALWAYS re-windows the result itself (visibility filter + `slice`), so a provider + * that honors `opts` should early-stop and return the `[0, limit)` PREFIX (it must + * NOT pre-apply `offset`). The JS index ignores `opts` and returns all matches — + * so honoring it is a pure native-side optimization that removes the O(N) FFI + * marshal at billion scale. Pairs with {@link getIdSetForFilter}. + */ + getIdsForFilter(filter: any, opts?: { limit?: number; offset?: number }): Promise /** * @description OPTIONAL: resolve a `where` filter to its matching id universe as * an {@link OpaqueIdSet} (a serialized roaring `Buffer`) WITHOUT materializing @@ -171,6 +182,17 @@ export interface MetadataIndexProvider { getAllVFSEntityCounts(): Promise> detectAndRepairCorruption(): Promise + /** + * OPTIONAL cheap (O(1)) cold-open consistency probe — the counterpart of the + * graph cold-load guard. Returns `true` if a sampled `(field, value, int)` is + * clean, `false` if it detects the cross-bucket phantom signature (an int whose + * current value for `field` no longer equals `value`). Brainy calls it ONCE per + * brain on the first read and, on `false`, runs {@link detectAndRepairCorruption} + * to self-heal — so an already-poisoned index repairs itself on open without the + * cost of the full-scan {@link validateConsistency}. A provider that omits it is + * simply never auto-probed (no behavior change). + */ + probeConsistency?(): Promise validateConsistency(): Promise<{ healthy: boolean avgEntriesPerEntity: number diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 1bae541a..069f6386 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1788,9 +1788,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { } /** - * Get IDs matching Brainy Field Operator metadata filter using indexes where possible + * Get IDs matching a Brainy Field Operator metadata filter using indexes where possible. + * The optional `_opts` page bound is part of the provider contract for the native + * index (early-stop at `offset+limit`); the JS index returns ALL matches and lets + * the caller window them, so `_opts` is intentionally ignored here. */ - async getIdsForFilter(filter: any): Promise { + async getIdsForFilter(filter: any, _opts?: { limit?: number; offset?: number }): Promise { if (!filter || Object.keys(filter).length === 0) { return [] } diff --git a/tests/unit/brainy/metadata-provider-contract.test.ts b/tests/unit/brainy/metadata-provider-contract.test.ts new file mode 100644 index 00000000..7e690978 --- /dev/null +++ b/tests/unit/brainy/metadata-provider-contract.test.ts @@ -0,0 +1,90 @@ +/** + * @module tests/unit/brainy/metadata-provider-contract + * @description Brainy-side wiring of the two metadata-provider contract additions + * confirmed with cor for the lockstep: + * + * 1. `probeConsistency()` — an OPTIONAL O(1) cold-open consistency sampler. On the + * first read, brainy calls it once; on `false` it self-heals via + * `detectAndRepairCorruption()` (the metadata counterpart of the graph cold-load + * guard). The native provider implements it; the JS index omits it (no-op). + * 2. `getIdsForFilter(filter, opts?)` — brainy passes a page bound on the UNSORTED + * `find({ type, where, limit })` path so a native provider can early-stop. The JS + * index ignores `opts`. + * + * These are unit tests of brainy's CALL behaviour (the real end-to-end honoring is + * exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS + * metadata index, which has neither method by default. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' + +describe('metadata-provider contract wiring (probeConsistency + getIdsForFilter opts)', () => { + let brain: Brainy + let mi: any + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await brain.init() + await brain.add({ data: 'a', type: NounType.Thing, metadata: { kind: 'x' } }) + await brain.add({ data: 'b', type: NounType.Thing, metadata: { kind: 'y' } }) + mi = (brain as any).metadataIndex + ;(brain as any)._metadataConsistencyProbed = false // reset the one-shot guard + }) + + it('calls probeConsistency once on cold open and self-heals via detectAndRepairCorruption on false', async () => { + let probes = 0 + let repairs = 0 + mi.probeConsistency = async () => { probes++; return false } // corrupt → must repair + const origRepair = mi.detectAndRepairCorruption.bind(mi) + mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } + + await brain.find({ where: { kind: 'x' } }) + expect(probes).toBe(1) + expect(repairs).toBe(1) + + // Second read must NOT re-probe (once per brain). + await brain.find({ where: { kind: 'y' } }) + expect(probes).toBe(1) + expect(repairs).toBe(1) + }) + + it('does NOT repair when the probe reports healthy', async () => { + let repairs = 0 + mi.probeConsistency = async () => true // clean + const origRepair = mi.detectAndRepairCorruption.bind(mi) + mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } + + await brain.find({ where: { kind: 'x' } }) + expect(repairs).toBe(0) + }) + + it('a probe failure never breaks the read (best-effort, retried next time)', async () => { + let probes = 0 + mi.probeConsistency = async () => { probes++; throw new Error('probe boom') } + + // The read still succeeds despite the throwing probe. + const rows = await brain.find({ where: { kind: 'x' } }) + expect(rows.length).toBe(1) + expect(probes).toBe(1) + // Guard reset on failure → the next read retries the probe. + await brain.find({ where: { kind: 'y' } }) + expect(probes).toBe(2) + }) + + it('passes a page bound to getIdsForFilter on the unsorted find path (offset 0, brainy re-windows)', async () => { + let seenOpts: { limit?: number; offset?: number } | undefined + const orig = mi.getIdsForFilter.bind(mi) + mi.getIdsForFilter = async (filter: any, opts?: { limit?: number; offset?: number }) => { + seenOpts = opts + return orig(filter, opts) + } + + const rows = await brain.find({ where: { kind: 'x' }, limit: 5 }) + expect(rows.length).toBe(1) // result correctness preserved (JS ignores opts) + expect(seenOpts).toBeDefined() + expect(typeof seenOpts!.limit).toBe('number') + expect(seenOpts!.limit).toBeGreaterThanOrEqual(5) + expect(seenOpts!.offset).toBe(0) // brainy applies offset itself via slice + }) +})