From 9dc4c5e62be9dc894c945893c8638bc9255d87e4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 2 Jul 2026 10:28:04 -0700 Subject: [PATCH] =?UTF-8?q?fix:=20metadata=20cold-read=20guard=20=E2=80=94?= =?UTF-8?q?=20no=20more=20silent=20[]=20on=20cold=20find({where})=20(7.33.?= =?UTF-8?q?5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to 7.33.4's graph cold-read guard, now for the metadata field index. A downstream deployment reported find({where}) returning a silent [] on a freshly-opened brain (a native metadata provider that reports data but hasn't loaded its field postings), blanking filtered pages after every restart. verifyMetadataLive() — one-shot per brain on the first filtered find(): probe a known persisted field value; if served, live (one O(1) probe on a warm brain). If not, rebuild from canonical + re-probe; if still unserved, throw the new exported MetadataIndexNotReadyError rather than let a silent [] pass. Inconclusive cases (empty store, no plain field, shared-store foreign entity) resolve to live, no false rebuild. Also exports BrainyError + GraphIndexNotReadyError (were internal). The JS index cold-loads correctly; this guards the native path (durable cure is cortex-side, same shape as the graph 2.7.8 cure). 4 unit tests. tsc 0, guard 4/4, full unit 1538 pass (+1 pre-existing flaky VFS perf test, green in isolation). --- RELEASES.md | 28 +++++ src/brainy.ts | 123 +++++++++++++++++++- src/errors/brainyError.ts | 24 ++++ src/index.ts | 5 + tests/unit/metadata-cold-read-guard.test.ts | 89 ++++++++++++++ 5 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 tests/unit/metadata-cold-read-guard.test.ts diff --git a/RELEASES.md b/RELEASES.md index 0ab71c45..96a5fb60 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,34 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so --- +## v7.33.5 — 2026-07-02 + +**Affected products:** any consumer whose brain restarts cold (every deploy) and issues +`find({ where: { field } })` filtered reads against a **native metadata index** — where a cold +open could return `[]` for a field lookup that HAS matching data, silently blanking filtered +pages until the index warmed. Drop-in; no API or data changes. + +### Fix — filtered `find({ where })` no longer silently returns `[]` on a cold open + +The companion to 7.33.2's graph cold-read guard, now for the **metadata field index**. On a fresh +brain open, a native metadata provider can report that data exists but not yet serve its `where` +postings — so `find({ where: { status: 'active' } })` returned an empty array **indistinguishable +from "no such data"**, and callers rendered "nothing found" over real data. + +Brainy now runs a one-time serving check on the first filtered `find()`: it probes a known +persisted field value and, if the index doesn't return the known entity, **rebuilds the index from +the canonical records** and re-probes. If it still cannot serve, it throws a loud, catchable +`MetadataIndexNotReadyError` (now exported, alongside `GraphIndexNotReadyError` and `BrainyError`) +rather than let a silent `[]` stand. On a warm brain the only cost is one O(1) probe. + +**What you get:** cold `where` reads are correct-by-default (self-healed) or fail loudly — never a +silent empty result. Catch `MetadataIndexNotReadyError` (or check `error.type === +'METADATA_INDEX_NOT_READY'`) to render a "warming up / retry" state instead of a false "no results". +The durable native cold-load cure remains a cortex-side follow-up; this guard makes silent-empty +impossible regardless of provider. + +--- + ## v7.33.2 — 2026-06-24 **Affected products:** any consumer using graph traversals — `find({ connected: { from, via } })`, diff --git a/src/brainy.ts b/src/brainy.ts index 97a2e055..4a8af3d9 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -48,7 +48,7 @@ import { MmapVectorBackend } from './hnsw/mmapVectorBackend.js' import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' import { RevisionConflictError } from './transaction/RevisionConflictError.js' -import { GraphIndexNotReadyError } from './errors/brainyError.js' +import { GraphIndexNotReadyError, MetadataIndexNotReadyError } from './errors/brainyError.js' import { ValidationConfig, validateAddParams, @@ -194,6 +194,8 @@ export class Brainy implements BrainyInterface { private _vfsInitialized = false // Track VFS init completion separately private _graphAdjacencyVerified = false // graph-adjacency cold-load consistency: verified-live this session private _graphAdjacencyVerifying = false // re-entrancy guard: a verify (rebuild→reads) is in flight + private _metadataVerified = false // metadata field-index cold-read guard: verified-serving this session + private _metadataVerifying = false // re-entrancy guard for verifyMetadataLive private _hub?: IntegrationHub // Integration Hub for external tools private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets private _aggregationIndex?: AggregationIndex // Incremental aggregation engine @@ -3224,6 +3226,15 @@ export class Brainy implements BrainyInterface { const hasFilterCriteria = params.where || params.type || params.subtype || params.service const hasGraphCriteria = params.connected + // Metadata cold-read guard: before trusting a filter result, verify the + // field index actually serves a known persisted value (one-shot per brain). + // A cold native index that has not loaded its `where` postings self-heals + // here (rebuild) or throws MetadataIndexNotReadyError — never a silent []. + // The graph counterpart (verifyGraphAdjacencyLive) covers `connected`. + if (hasFilterCriteria) { + await this.verifyMetadataLive() + } + // Handle metadata-only queries (no vector search needed) if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) { // Build filter for metadata index @@ -8258,6 +8269,116 @@ export class Brainy implements BrainyInterface { } } + /** + * The metadata field-index counterpart of {@link verifyGraphAdjacencyLive}. + * On a cold open a native metadata provider can report data yet not serve its + * `where` postings, so `find({ where })` silently returns `[]` — a downstream + * deployment reported cold filtered reads blanking pages after every restart. + * This one-shot guard, run on the first FILTERED `find()`, takes a KNOWN + * persisted entity + one of its plain field values and asks the index to + * resolve it. If the known id comes back the postings are live (the common + * case, and the only cost on a warm brain — one O(1) probe). If not, they did + * not load: brainy rebuilds from the canonical records and re-probes, and + * raises a loud {@link MetadataIndexNotReadyError} only if the rebuild still + * cannot serve — never a silent `[]` over existing data. Inconclusive cases + * (empty store, no plain field to probe, a shared store surfacing a foreign + * entity) resolve to live — never a false rebuild. + */ + private async verifyMetadataLive(): Promise<'live' | 'rebuilt'> { + if (this._metadataVerified) return 'live' + // Re-entrancy: rebuild() can trigger reads that call back into this guard. + if (this._metadataVerifying) return 'live' + this._metadataVerifying = true + try { + // A KNOWN persisted entity + one plain field to probe. Sample a few so a + // system-only entity (e.g. the VFS root) doesn't make every open inconclusive. + const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) + let probe: { field: string; value: string | number | boolean; id: string } | null = null + for (const noun of sample.items ?? []) { + probe = this.pickMetadataProbe(noun as { id?: string; metadata?: Record }) + if (probe) break + } + if (!probe) { + this._metadataVerified = true + return 'live' // empty store, or nothing with a plain user field to probe + } + const p = probe + + const probeServes = async (): Promise => { + try { + const ids = await this.metadataIndex.getIdsForFilter({ [p.field]: p.value }) + return ids.includes(p.id) + } catch { + // FIELD_NOT_INDEXED for a field a persisted entity actually holds is + // itself the cold/broken signal — treat as not-serving (→ rebuild). + return false + } + } + + if (await probeServes()) { + this._metadataVerified = true + return 'live' // field postings are live — the common case + } + + if (!this.config.silent) { + console.warn( + `[Brainy] Metadata field index returns no match for a known persisted value of ` + + `'${p.field}' — the field postings did not load on open. Rebuilding from storage…` + ) + } + await this.metadataIndex.rebuild() + + if (await probeServes()) { + this._metadataVerified = true + return 'rebuilt' + } + throw new MetadataIndexNotReadyError( + `Metadata field index cannot serve a known persisted value of '${p.field}' even after ` + + `a rebuild — find({ where }) and other filtered reads cannot be served reliably for ` + + `this brain (a silent empty result would misrepresent existing data).` + ) + } catch (err) { + if (err instanceof MetadataIndexNotReadyError) throw err + // A transient probe/rebuild failure must not break the query NOR mask as + // "no data". Allow a re-check on the next filtered read and fall through. + this._metadataVerified = false + if (!this.config.silent) { + console.warn(`[Brainy] Metadata consistency check skipped (transient): ${err}`) + } + return 'live' + } finally { + this._metadataVerifying = false + } + } + + /** + * Choose one plain (scalar, user-written) field from an entity's metadata to + * probe the field index with — skipping internal / system fields (`__words__` + * text hash, VFS markers, reserved `visibility`/`subtype`/`service`, the `noun` + * type alias, timestamps) that use different index paths, and any non-scalar + * value. Returns `null` when the entity has no probeable field. + */ + private pickMetadataProbe( + noun: { id?: string; metadata?: Record } + ): { field: string; value: string | number | boolean; id: string } | null { + const id = noun?.id + const metadata = noun?.metadata + if (!id || !metadata || typeof metadata !== 'object') return null + const skip = new Set([ + 'noun', 'type', 'subtype', 'service', 'visibility', 'id', 'vector', + 'isVFSEntity', 'vfsType', 'vfsPath', 'vfsName', 'createdAt', 'updatedAt' + ]) + for (const [field, value] of Object.entries(metadata)) { + if (field.startsWith('_') || skip.has(field)) continue + if (value === null || value === undefined) continue + const t = typeof value + if (t === 'string' || t === 'number' || t === 'boolean') { + return { field, value: value as string | number | boolean, id } + } + } + return null + } + async neighbors( entityId: string, options?: { diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index 4bc75902..77183f50 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -12,6 +12,7 @@ export type BrainyErrorType = | 'VALIDATION' | 'FIELD_NOT_INDEXED' | 'GRAPH_INDEX_NOT_READY' + | 'METADATA_INDEX_NOT_READY' /** * Custom error class for Brainy operations @@ -248,3 +249,26 @@ export class GraphIndexNotReadyError extends BrainyError { } } } + +/** + * Thrown when the metadata field index reports data but cannot serve a KNOWN + * persisted field value even after a rebuild — i.e. the `where` / filter + * postings did not load on a cold open and could not be restored. The + * field-index counterpart of {@link GraphIndexNotReadyError}: it replaces the + * silent-empty failure mode (a cold `find({ where })` returning `[]` + * indistinguishable from "no such data") with a loud, catchable error, so a + * consumer never renders "nothing found" over data that is simply not-yet-warm. + * + * Detected once per brain by a known-value serving probe on the first filtered + * `find()`; brainy self-heals (rebuilds the index from the canonical records) + * first and only raises this if the rebuild still cannot serve the known value. + */ +export class MetadataIndexNotReadyError extends BrainyError { + constructor(message: string, originalError?: Error) { + super(message, 'METADATA_INDEX_NOT_READY', false, originalError) + this.name = 'MetadataIndexNotReadyError' + if (Error.captureStackTrace) { + Error.captureStackTrace(this, MetadataIndexNotReadyError) + } + } +} diff --git a/src/index.ts b/src/index.ts index 8192fa15..ebc46561 100644 --- a/src/index.ts +++ b/src/index.ts @@ -139,6 +139,11 @@ export type { Migration, MigrationState, MigrationPreview, MigrationResult, Migr // Export optimistic-concurrency types (7.31.0) export { RevisionConflictError } from './transaction/RevisionConflictError.js' +// Cold-read guards (7.33.4 graph / 7.33.5 metadata): catch these instead of +// silently rendering an empty cold-index result. `error.type` also works. +export { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError } from './errors/brainyError.js' +export type { BrainyErrorType } from './errors/brainyError.js' + // Export embedding functionality import { UniversalSentenceEncoder, diff --git a/tests/unit/metadata-cold-read-guard.test.ts b/tests/unit/metadata-cold-read-guard.test.ts new file mode 100644 index 00000000..75e3397a --- /dev/null +++ b/tests/unit/metadata-cold-read-guard.test.ts @@ -0,0 +1,89 @@ +/** + * Metadata cold-read guard (verifyMetadataLive) — 7.33.5 hotfix. + * + * A downstream deployment reported cold `find({ where })` returning a silent `[]` + * on a freshly-opened brain (a native metadata index that reports data but has + * not loaded its field postings), blanking filtered pages after every restart. + * This guard, the field-index counterpart of verifyGraphAdjacencyLive (7.33.4), + * probes a known persisted value on the first filtered find(): if the index does + * not serve it, brainy rebuilds and re-probes, and raises a loud + * MetadataIndexNotReadyError only if the rebuild still can't serve — never a + * silent empty result that misrepresents existing data. + * + * The JS index cold-loads correctly, so we simulate the cold native failure mode + * by intercepting the provider's getIdsForFilter/rebuild. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy, NounType, MetadataIndexNotReadyError } from '../../src/index.js' + +const V = () => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001) + +describe('Metadata cold-read guard (7.33.5)', () => { + let brain: any + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'active' } }) + await brain.add({ vector: V(), type: NounType.Concept, metadata: { status: 'archived' } }) + await brain.flush() + }) + + it('warm brain: filtered find is correct and the guard does not rebuild', async () => { + const mi = brain.metadataIndex + let rebuilds = 0 + const origRebuild = mi.rebuild.bind(mi) + mi.rebuild = async () => { + rebuilds++ + return origRebuild() + } + const res = await brain.find({ where: { status: 'active' }, limit: 100 }) + expect(res.length).toBe(1) + expect(rebuilds).toBe(0) + expect(brain._metadataVerified).toBe(true) + mi.rebuild = origRebuild + }) + + it('cold index: verifyMetadataLive self-heals via rebuild — find({where}) is correct, NOT silent []', async () => { + const mi = brain.metadataIndex + const origGetIds = mi.getIdsForFilter.bind(mi) + const origRebuild = mi.rebuild.bind(mi) + let cold = true + brain._metadataVerified = false + mi.getIdsForFilter = async (...a: any[]) => (cold ? [] : origGetIds(...a)) + mi.rebuild = async () => { + await origRebuild() + cold = false + } + try { + const res = await brain.find({ where: { status: 'active' }, limit: 100 }) + expect(res.length).toBe(1) + } finally { + mi.getIdsForFilter = origGetIds + mi.rebuild = origRebuild + } + }) + + it('unrecoverably cold index: find({where}) throws MetadataIndexNotReadyError — never a silent []', async () => { + const mi = brain.metadataIndex + const origGetIds = mi.getIdsForFilter.bind(mi) + const origRebuild = mi.rebuild.bind(mi) + brain._metadataVerified = false + mi.getIdsForFilter = async () => [] + mi.rebuild = async () => {} + try { + await expect(brain.find({ where: { status: 'active' }, limit: 100 })).rejects.toBeInstanceOf( + MetadataIndexNotReadyError + ) + } finally { + mi.getIdsForFilter = origGetIds + mi.rebuild = origRebuild + } + }) + + it('a query with no filter does not trigger the metadata probe', async () => { + brain._metadataVerified = false + await brain.find({ vector: V(), limit: 5 }) + expect(brain._metadataVerified).toBe(false) + }) +})