fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds
Two cures from the pair's first production adoption, both measured live.
THE READ GATE IS PER-FAMILY. The report-driven gate refused on ANY
provider's not-ready verdict at every read choke point — so a pure
metadata find({ where }) was refused because the VECTOR leg was not
serving, and a deployment's badge reads returned errors for a verdict that
had nothing to do with them. A read may only be refused by the family it
actually consults: metadata reads by the metadata leg (plus graph for a
`connected` filter), vector search by the vector leg, traversal by the
graph leg. Callers name what they need; the existing narration-once-per-
generation and typed-refusal laws are unchanged within a family.
NO RE-EMBED ON UNCHANGED DATA. update() — and its transact() planner —
treated any write that carried `data` as a data change: with
deferEmbedding it queued a landing, and the worker re-embedded and re-landed
a vector for content that had not changed. A host heartbeat re-writing an
unchanged row every few seconds therefore fed a live index-row loop on a
production store. A write carrying the row's current data (structural
compare, key order normalized) is now not a data change: no re-embed, no
deferred landing, no vector rewrite; the metadata write itself still
commits. A real change re-embeds exactly as before.
Pinned in tests/integration/read-gate-scope-and-no-reembed.test.ts — both
pins red-proved against the unfixed code with the production shapes
verbatim. Two health-gate pins that encoded the old brain-global scope are
re-pointed to the family their reads consult.
This commit is contained in:
parent
21e506e802
commit
c039411e08
3 changed files with 152 additions and 13 deletions
|
|
@ -3598,7 +3598,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// re-embed below — a stale vector left behind with no path to ever
|
// re-embed below — a stale vector left behind with no path to ever
|
||||||
// correct itself (a quiet loss, not the deferred-but-eventually-
|
// correct itself (a quiet loss, not the deferred-but-eventually-
|
||||||
// correct flicker the deferEmbedding contract promises).
|
// correct flicker the deferEmbedding contract promises).
|
||||||
const hasNewData = params.data !== undefined && params.data !== null
|
const rawHasNewData = params.data !== undefined && params.data !== null
|
||||||
|
// NO RE-EMBED ON UNCHANGED DATA: a write carrying the row's CURRENT data
|
||||||
|
// is not a data change — no re-embed, no deferred landing, no vector
|
||||||
|
// rewrite. A host heartbeat re-writing an unchanged row every few
|
||||||
|
// seconds fed a live index-row loop on a production store (each
|
||||||
|
// "change" landed a vector); the amplifier dies here regardless of how
|
||||||
|
// often the host writes.
|
||||||
|
const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data)
|
||||||
|
const hasNewData = rawHasNewData && !dataUnchanged
|
||||||
// MT5 deferred re-embedding: the OLD vector keeps serving semantic
|
// MT5 deferred re-embedding: the OLD vector keeps serving semantic
|
||||||
// search — stale-but-present, never absent (the flicker law) — until
|
// search — stale-but-present, never absent (the flicker law) — until
|
||||||
// the background worker embeds the new data and swaps it atomically.
|
// the background worker embeds the new data and swaps it atomically.
|
||||||
|
|
@ -4209,7 +4217,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// writes while every non-find() read served empty from a not-ready
|
// writes while every non-find() read served empty from a not-ready
|
||||||
// provider for 15 minutes. A CHECK only — it never builds; throws a typed
|
// provider for 15 minutes. A CHECK only — it never builds; throws a typed
|
||||||
// NotReady error if a provider's health report says it isn't serving.
|
// NotReady error if a provider's health report says it isn't serving.
|
||||||
this.ensureIndexesLoaded()
|
this.ensureIndexesLoaded(['graph'])
|
||||||
const entityInt = this.graphEntityInt(uuid)
|
const entityInt = this.graphEntityInt(uuid)
|
||||||
if (entityInt === undefined) return []
|
if (entityInt === undefined) return []
|
||||||
const neighborInts = await this.graphIndex.getNeighbors(entityInt, options)
|
const neighborInts = await this.graphIndex.getNeighbors(entityInt, options)
|
||||||
|
|
@ -6803,7 +6811,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// READ-SURFACE READINESS GATE (see filterIdsBelted): a CHECK only — it
|
// READ-SURFACE READINESS GATE (see filterIdsBelted): a CHECK only — it
|
||||||
// never builds. open() already brought every provider to serving before
|
// never builds. open() already brought every provider to serving before
|
||||||
// init() returned; this throws a typed NotReady error if one isn't.
|
// init() returned; this throws a typed NotReady error if one isn't.
|
||||||
this.ensureIndexesLoaded()
|
this.ensureIndexesLoaded(['metadata'])
|
||||||
|
|
||||||
// Loudly flag a degraded derived index (failed init rebuild, or an
|
// Loudly flag a degraded derived index (failed init rebuild, or an
|
||||||
// adopt-forward degraded commit) so a partial result is never mistaken for
|
// adopt-forward degraded commit) so a partial result is never mistaken for
|
||||||
|
|
@ -6815,6 +6823,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
let params: FindParams<T> =
|
let params: FindParams<T> =
|
||||||
typeof query === 'string' ? await this.parseNaturalQuery(query) : query
|
typeof query === 'string' ? await this.parseNaturalQuery(query) : query
|
||||||
|
|
||||||
|
// The vector and graph legs gate only the finds that consult them.
|
||||||
|
const consultsVector = Boolean(
|
||||||
|
(params.query && params.query.trim() !== '') || params.vector || params.near
|
||||||
|
)
|
||||||
|
if (consultsVector) this.ensureIndexesLoaded(['vector'])
|
||||||
|
if (params.connected) this.ensureIndexesLoaded(['graph'])
|
||||||
|
|
||||||
// Id normalization (8.0): resolve the graph-traversal anchor id(s) so a
|
// Id normalization (8.0): resolve the graph-traversal anchor id(s) so a
|
||||||
// caller may constrain by natural key. Each maps to the canonical UUID
|
// caller may constrain by natural key. Each maps to the canonical UUID
|
||||||
// add() stored; real UUIDs pass through. Done once here so every downstream
|
// add() stored; real UUIDs pass through. Done once here so every downstream
|
||||||
|
|
@ -10663,7 +10678,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// content (see the identical hasNewData in update()); a plain truthy
|
// content (see the identical hasNewData in update()); a plain truthy
|
||||||
// check would silently skip re-embedding an emptied value and leave a
|
// check would silently skip re-embedding an emptied value and leave a
|
||||||
// stale vector with no path to ever correct itself.
|
// stale vector with no path to ever correct itself.
|
||||||
const hasNewData = params.data !== undefined && params.data !== null
|
const rawHasNewData = params.data !== undefined && params.data !== null
|
||||||
|
// No re-embed on unchanged data — the transact() mirror of update()'s rule.
|
||||||
|
const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data)
|
||||||
|
const hasNewData = rawHasNewData && !dataUnchanged
|
||||||
let vector = existing.vector
|
let vector = existing.vector
|
||||||
if (params.vector) {
|
if (params.vector) {
|
||||||
if (this.dimensions && params.vector.length !== this.dimensions) {
|
if (this.dimensions && params.vector.length !== this.dimensions) {
|
||||||
|
|
@ -12158,7 +12176,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// writes while every non-find() read served empty from a not-ready
|
// writes while every non-find() read served empty from a not-ready
|
||||||
// provider for 15 minutes. A CHECK only — it never builds; throws a typed
|
// provider for 15 minutes. A CHECK only — it never builds; throws a typed
|
||||||
// NotReady error if a provider's health report says it isn't serving.
|
// NotReady error if a provider's health report says it isn't serving.
|
||||||
this.ensureIndexesLoaded()
|
this.ensureIndexesLoaded(['metadata'])
|
||||||
try {
|
try {
|
||||||
return await this.metadataIndex.getIdsForFilter(filter, opts)
|
return await this.metadataIndex.getIdsForFilter(filter, opts)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
@ -14654,7 +14672,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// writes while every non-find() read served empty from a not-ready
|
// writes while every non-find() read served empty from a not-ready
|
||||||
// provider for 15 minutes. A CHECK only — it never builds; throws a typed
|
// provider for 15 minutes. A CHECK only — it never builds; throws a typed
|
||||||
// NotReady error if a provider's health report says it isn't serving.
|
// NotReady error if a provider's health report says it isn't serving.
|
||||||
this.ensureIndexesLoaded()
|
this.ensureIndexesLoaded(['graph'])
|
||||||
// 8.0 BigInt boundary: unmapped node → no relations.
|
// 8.0 BigInt boundary: unmapped node → no relations.
|
||||||
const nodeInt = this.graphEntityInt(nodeId)
|
const nodeInt = this.graphEntityInt(nodeId)
|
||||||
if (nodeInt === undefined) return []
|
if (nodeInt === undefined) return []
|
||||||
|
|
@ -16543,12 +16561,47 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* `prodLog.warn` ONCE per (provider, `report.generation`) — never once per
|
* `prodLog.warn` ONCE per (provider, `report.generation`) — never once per
|
||||||
* read — before any throw decision is made.
|
* read — before any throw decision is made.
|
||||||
*/
|
*/
|
||||||
private ensureIndexesLoaded(): void {
|
/**
|
||||||
const providers: ReadonlyArray<readonly [string, unknown, new (message: string, originalError?: Error) => BrainyError]> = [
|
* @description Whether two entity `data` payloads are the same content —
|
||||||
|
* the "no re-embed on unchanged data" comparison. Primitives compare by
|
||||||
|
* value; objects compare structurally with key order normalized.
|
||||||
|
* @param a - The incoming data.
|
||||||
|
* @param b - The stored data.
|
||||||
|
* @returns `true` when the content is identical.
|
||||||
|
*/
|
||||||
|
private static sameEntityData(a: unknown, b: unknown): boolean {
|
||||||
|
if (a === b) return true
|
||||||
|
if (a === null || b === null || typeof a !== typeof b) return false
|
||||||
|
if (typeof a !== 'object') return false
|
||||||
|
const stable = (v: unknown): string =>
|
||||||
|
JSON.stringify(v, (_k, val) =>
|
||||||
|
val && typeof val === 'object' && !Array.isArray(val)
|
||||||
|
? Object.keys(val as Record<string, unknown>).sort().reduce((o, k) => {
|
||||||
|
;(o as Record<string, unknown>)[k] = (val as Record<string, unknown>)[k]
|
||||||
|
return o
|
||||||
|
}, {} as Record<string, unknown>)
|
||||||
|
: val
|
||||||
|
)
|
||||||
|
try { return stable(a) === stable(b) } catch { return false }
|
||||||
|
}
|
||||||
|
|
||||||
|
private ensureIndexesLoaded(
|
||||||
|
families: ReadonlyArray<'vector' | 'metadata' | 'graph'> = ['vector', 'metadata', 'graph']
|
||||||
|
): void {
|
||||||
|
// PER-FAMILY SCOPE. This gate used to refuse on ANY provider's not-ready
|
||||||
|
// verdict at every read choke point — so a pure metadata find({where})
|
||||||
|
// was refused because the VECTOR leg was not serving; a production
|
||||||
|
// deployment's badge reads returned 500s for exactly that reason on the
|
||||||
|
// pair's first adoption. A read may only be refused by the family it
|
||||||
|
// actually consults: metadata reads by the metadata leg (+ graph for a
|
||||||
|
// `connected` filter), vector search by the vector leg, traversal by the
|
||||||
|
// graph leg. Callers name what they need.
|
||||||
|
const all: ReadonlyArray<readonly ['vector' | 'metadata' | 'graph', unknown, new (message: string, originalError?: Error) => BrainyError]> = [
|
||||||
['vector', this.index, VectorIndexNotReadyError],
|
['vector', this.index, VectorIndexNotReadyError],
|
||||||
['metadata', this.metadataIndex, MetadataIndexNotReadyError],
|
['metadata', this.metadataIndex, MetadataIndexNotReadyError],
|
||||||
['graph', this.graphIndex, GraphIndexNotReadyError]
|
['graph', this.graphIndex, GraphIndexNotReadyError]
|
||||||
]
|
]
|
||||||
|
const providers = all.filter(([name]) => families.includes(name))
|
||||||
|
|
||||||
for (const [name, provider, ErrorClass] of providers) {
|
for (const [name, provider, ErrorClass] of providers) {
|
||||||
// Migration LOCK (#18) deference: a migrating provider owns its own
|
// Migration LOCK (#18) deference: a migrating provider owns its own
|
||||||
|
|
|
||||||
|
|
@ -186,6 +186,8 @@ describe('health gate (b) — unledgered is unknown: never blocks a serving prov
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('health gate (c) — degraded-but-serving narrates once per generation', () => {
|
describe('health gate (c) — degraded-but-serving narrates once per generation', () => {
|
||||||
|
// PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the
|
||||||
|
// degraded report lives on the family the read actually consults.
|
||||||
it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => {
|
it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => {
|
||||||
const brain = new Brainy(createTestConfig({ silent: true }))
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -195,7 +197,7 @@ describe('health gate (c) — degraded-but-serving narrates once per generation'
|
||||||
|
|
||||||
const internals = internalsOf(brain)
|
const internals = internalsOf(brain)
|
||||||
let generation = 1
|
let generation = 1
|
||||||
internals.index.healthReport = () =>
|
internals.metadataIndex.healthReport = () =>
|
||||||
healthReport({
|
healthReport({
|
||||||
provider: 'vector',
|
provider: 'vector',
|
||||||
serving: true,
|
serving: true,
|
||||||
|
|
@ -216,7 +218,7 @@ describe('health gate (c) — degraded-but-serving narrates once per generation'
|
||||||
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1)
|
||||||
expect(countNarrations()).toBe(2) // generation bumped — a second narration
|
expect(countNarrations()).toBe(2) // generation bumped — a second narration
|
||||||
|
|
||||||
delete internals.index.healthReport
|
delete internals.metadataIndex.healthReport
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
@ -332,6 +334,7 @@ describe('health gate (f) — the ceremony door: explicit rebuild bypasses invar
|
||||||
})
|
})
|
||||||
|
|
||||||
describe('health gate (g) — a throwing healthReport() is a contract violation, never read as healthy', () => {
|
describe('health gate (g) — a throwing healthReport() is a contract violation, never read as healthy', () => {
|
||||||
|
// PER-FAMILY LAW (10.4.1): the throwing report sits on the family the read consults.
|
||||||
it('healthReport() that throws refuses loudly with the typed NotReady error naming the throw', async () => {
|
it('healthReport() that throws refuses loudly with the typed NotReady error naming the throw', async () => {
|
||||||
const brain = new Brainy(createTestConfig({ silent: true }))
|
const brain = new Brainy(createTestConfig({ silent: true }))
|
||||||
await brain.init()
|
await brain.init()
|
||||||
|
|
@ -340,13 +343,13 @@ describe('health gate (g) — a throwing healthReport() is a contract violation,
|
||||||
await brain.flush()
|
await brain.flush()
|
||||||
|
|
||||||
const internals = internalsOf(brain)
|
const internals = internalsOf(brain)
|
||||||
internals.index.healthReport = () => {
|
internals.metadataIndex.healthReport = () => {
|
||||||
throw new Error('accelerator: mmap window busy')
|
throw new Error('accelerator: mmap window busy')
|
||||||
}
|
}
|
||||||
|
|
||||||
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(VectorIndexNotReadyError)
|
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError)
|
||||||
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/mmap window busy/)
|
await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/mmap window busy/)
|
||||||
|
|
||||||
delete internals.index.healthReport
|
delete internals.metadataIndex.healthReport
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
||||||
83
tests/integration/read-gate-scope-and-no-reembed.test.ts
Normal file
83
tests/integration/read-gate-scope-and-no-reembed.test.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/read-gate-scope-and-no-reembed
|
||||||
|
* @description Two cures from the pair's first production adoption:
|
||||||
|
* (1) THE READ GATE IS PER-FAMILY — a not-serving VECTOR leg refuses vector
|
||||||
|
* search only; a pure metadata find({ where }) and graph traversal keep
|
||||||
|
* serving. The brain-global gate refused a deployment's badge reads for a
|
||||||
|
* vector-leg verdict that had nothing to do with them.
|
||||||
|
* (2) NO RE-EMBED ON UNCHANGED DATA — an update() carrying the row's current
|
||||||
|
* data lands no vector, defers no embed, rewrites nothing. A host
|
||||||
|
* heartbeat re-writing an unchanged row fed a live index-row loop.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { Brainy, VectorIndexNotReadyError } from '../../src/index.js'
|
||||||
|
|
||||||
|
describe('read gate scope + no re-embed on unchanged data', () => {
|
||||||
|
let dir: string
|
||||||
|
let brain: any
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||||
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-gate-scope-'))
|
||||||
|
brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
|
||||||
|
await brain.init()
|
||||||
|
})
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close?.().catch(() => {})
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a not-serving VECTOR leg refuses vector search only — metadata and graph reads keep serving', async () => {
|
||||||
|
const a = await brain.add({ data: 'employee alpha', type: 'person', metadata: { status: 'active' } })
|
||||||
|
const b = await brain.add({ data: 'employee beta', type: 'person', metadata: { status: 'active' } })
|
||||||
|
await brain.relate({ from: a, to: b, type: 'relatedTo' })
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
// The vector provider says it is NOT serving (a rebuild-class failure).
|
||||||
|
brain.index.healthReport = () => ({
|
||||||
|
provider: 'vector', healthy: false, serving: false, generation: 7, unledgered: [],
|
||||||
|
invariants: [{ name: 'node-coverage', holds: false, heal: 'rebuild', detail: 'posted 0 < canonical 2', source: 'ledger' }],
|
||||||
|
checkedAt: 1, durationMs: 1
|
||||||
|
})
|
||||||
|
try {
|
||||||
|
const byStatus = await brain.find({ where: { status: 'active' } })
|
||||||
|
expect(byStatus.map((r: any) => r.id).sort(), 'metadata find serves').toEqual([a, b].sort())
|
||||||
|
const rel = await brain.related(a)
|
||||||
|
expect(rel.length, 'graph traversal serves').toBe(1)
|
||||||
|
await expect(brain.find({ query: 'employee' }), 'vector search refuses typed').rejects.toBeInstanceOf(VectorIndexNotReadyError)
|
||||||
|
} finally {
|
||||||
|
delete brain.index.healthReport
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('update() with the row\'s current data re-embeds nothing; a real change re-embeds', async () => {
|
||||||
|
const id = await brain.add({ data: 'invoice 1042 pending', type: 'document', metadata: { n: 1 } })
|
||||||
|
await brain.flush()
|
||||||
|
const before = (await brain.get(id, { includeVectors: true })).vector
|
||||||
|
const ledgerBefore = await brain.storage.getCanonicalCounts()
|
||||||
|
const logBefore = (await brain.transactionLog({ limit: 50 })).length
|
||||||
|
|
||||||
|
// The heartbeat shape: same data, re-written, deferred.
|
||||||
|
for (let i = 0; i < 3; i++) {
|
||||||
|
await brain.update({ id, data: 'invoice 1042 pending', metadata: { n: 1, tick: i }, deferEmbedding: true })
|
||||||
|
}
|
||||||
|
await brain.flush()
|
||||||
|
const after = (await brain.get(id, { includeVectors: true })).vector
|
||||||
|
const ledgerAfter = await brain.storage.getCanonicalCounts()
|
||||||
|
const log = await brain.transactionLog({ limit: 50 })
|
||||||
|
expect(after, 'vector untouched by unchanged-data writes').toEqual(before)
|
||||||
|
expect(ledgerAfter.vectors.all, 'vectored ledger untouched').toBe(ledgerBefore.vectors.all)
|
||||||
|
expect(log.filter((e: any) => e.origin === 'system:embed-landing').length, 'no landing commit for unchanged data').toBe(0)
|
||||||
|
expect(log.length - logBefore, 'the metadata writes themselves still commit').toBe(3)
|
||||||
|
|
||||||
|
// A REAL change re-embeds (deferred → the worker lands it).
|
||||||
|
await brain.update({ id, data: 'invoice 1042 PAID', deferEmbedding: true })
|
||||||
|
await brain.flush()
|
||||||
|
const changed = (await brain.get(id, { includeVectors: true })).vector
|
||||||
|
expect(changed, 'a real data change re-embeds').not.toEqual(before)
|
||||||
|
expect((await brain.storage.getCanonicalCounts()).vectors.all, 'a re-embed of a vectored row never double-counts').toBe(ledgerBefore.vectors.all)
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue