From 0c028dfc81c9b053bc9531f6a36b4d04350cb661 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 24 Aug 2026 09:50:35 -0700 Subject: [PATCH 001/150] feat(update-seam): first-class update operation through the provider seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brainy's planner emits a single UpdateInMetadataIndexOperation / UpdateVerbInGraphIndexOperation when the registered provider announces the `update-op` capability (capabilities has 'update-op' AND the method exists — both halves), and the legacy remove+add pair otherwise. A provider whose capability set claims what the instance lacks is refused at registration with a typed ProviderCapabilityMismatchError — never a silent fallback. - MetadataIndexProvider.updateIndex(id, before, after, generation?) and GraphIndexProvider.updateVerb(id, before, after, generation) — optional, rollback symmetric by construction (update(a,b) undone by update(b,a)). - Emission at update(), planTxUpdate() and updateRelation()'s graph leg. - transact() gains op:'updateRelation' (TxUpdateRelationOperation), born batchable; updateRelation()'s record build is shared with the planner. - Both paths live for one overlap release; the pair path retires with the provider-side compensation layer in the following cut. Pinned in tests/integration/update-op-emission.test.ts (7 pins). --- docs/api/README.md | 2 + src/brainy.ts | 301 ++++++++++--- src/db/db.ts | 64 ++- src/db/types.ts | 15 +- src/errors/brainyError.ts | 41 ++ src/index.ts | 2 +- src/plugin.ts | 80 ++++ src/transaction/operations/IndexOperations.ts | 106 +++++ src/transaction/operations/index.ts | 11 + .../operations/updateCapability.ts | 125 ++++++ tests/integration/update-op-emission.test.ts | 409 ++++++++++++++++++ 11 files changed, 1090 insertions(+), 66 deletions(-) create mode 100644 src/transaction/operations/updateCapability.ts create mode 100644 tests/integration/update-op-emission.test.ts diff --git a/docs/api/README.md b/docs/api/README.md index 4ca84364..f5860574 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -848,6 +848,7 @@ const db = await brain.transact([ { op: 'add', id: orderId, type: NounType.Document, subtype: 'order', data: 'Order #1042' }, { op: 'update', id: customerId, metadata: { lastOrderAt: Date.now() }, ifRev: customer._rev }, { op: 'relate', from: customerId, to: orderId, type: VerbType.Creates, subtype: 'purchase' }, + { op: 'updateRelation', id: purchaseRelationId, subtype: 'return' }, { op: 'remove', id: staleDraftId }, { op: 'unrelate', id: oldRelationId } ], { @@ -864,6 +865,7 @@ db.receipt.generation // the committed generation - `{ op: 'update', ... }` — same parameters as `update()`, including per-entity `ifRev` CAS - `{ op: 'remove', id }` — deletes the entity plus its relationships (same cascade as `delete()`) - `{ op: 'relate', ... }` — same parameters as `relate()`, including `bidirectional`; duplicates dedupe to the existing relationship id +- `{ op: 'updateRelation', ... }` — same parameters as `updateRelation()`; a batchable, first-class relationship update (not `unrelate` + `relate` — the relationship id and its edge never change) - `{ op: 'unrelate', id }` — deletes a relationship by id Operations may reference ids created earlier in the same batch. diff --git a/src/brainy.ts b/src/brainy.ts index 6ebce2fd..5a181535 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -103,7 +103,12 @@ import { UpdateNounMetadataOperation, UpdateVerbMetadataOperation, DeleteNounMetadataOperation, - DeleteVerbMetadataOperation + DeleteVerbMetadataOperation, + UpdateInMetadataIndexOperation, + UpdateVerbInGraphIndexOperation, + metadataUpdateOpProvider, + graphUpdateOpProvider, + assertUpdateCapabilityCoherent } from './transaction/operations/index.js' import { BaseOperationalMode, @@ -196,7 +201,7 @@ import { } from './events/changeFeed.js' import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' -import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' +import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError, ProviderCapabilityMismatchError } from './errors/brainyError.js' import { assessIndexReadiness } from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' import { asBrainyFieldRefusal } from './db/fieldAddressing.js' @@ -1276,6 +1281,11 @@ export class Brainy implements BrainyInterface { entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined, }) } + // Registration-time refusal (before any write can run): a provider + // whose `capabilities` set claims 'update-op' while its instance lacks + // `updateIndex` is a typed, loud refusal — never a silent fallback + // discovered only at the first write. + assertUpdateCapabilityCoherent(this.metadataIndex, 'metadata') // Provider: graph index factory const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') @@ -1290,6 +1300,9 @@ export class Brainy implements BrainyInterface { ]) this.graphIndex = graphIndex } + // Same registration-time refusal, graph half (see the metadata-index + // check above). + assertUpdateCapabilityCoherent(this.graphIndex, 'graph') // Fact-log v2 mint seam: after-image records carry minted dense ints, // and the ONE authority for those assignments is the metadata index's @@ -1682,6 +1695,14 @@ export class Brainy implements BrainyInterface { if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { throw error } + // Same rationale, provider-capability half: a registration-time refusal + // (a provider's `capabilities` set lies about implementing 'update-op') + // carries a machine-readable `.type`/`.family`/`.missingMethod` — the + // whole point of the typed-error family — so it must not be flattened + // into a message-only generic Error either. + if (error instanceof ProviderCapabilityMismatchError) { + throw error + } throw new Error(`Failed to initialize Brainy: ${error}`) } } @@ -3619,12 +3640,23 @@ export class Brainy implements BrainyInterface { createdBy: existing.createdBy, metadata: existing.metadata // CRITICAL: keep as nested 'metadata' property! } - tx.addOperation( - new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata, this.indexWriteGeneration) - ) - tx.addOperation( - new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing, this.indexWriteGeneration) - ) + // ONE atomic metadata-index leg when the provider announces the + // update-op capability (both halves of the check pass), else the + // legacy remove-old/add-new pair — the one-train overlap this + // release (see MetadataIndexProvider.updateIndex in src/plugin.ts). + const metadataUpdateProvider = metadataUpdateOpProvider(this.metadataIndex) + if (metadataUpdateProvider) { + tx.addOperation( + new UpdateInMetadataIndexOperation(metadataUpdateProvider, params.id, removalMetadata, entityForIndexing, this.indexWriteGeneration) + ) + } else { + tx.addOperation( + new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata, this.indexWriteGeneration) + ) + tx.addOperation( + new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing, this.indexWriteGeneration) + ) + } }, casPrecommit, this._changeFeed.hasListeners ? [ { @@ -4837,16 +4869,103 @@ export class Brainy implements BrainyInterface { validateUpdateRelationParams(params) - const existing = await this.storage.getVerb(params.id) if (!existing) { throw new RelationNotFoundError(params.id) } - // Legacy stored shapes carried the verb type under `type` instead of the - // canonical `verb` field — read both, canonical first. - const existingRec: HNSWVerbWithMetadata & { type?: VerbType } = existing - const newVerbType = params.type ?? existingRec.verb ?? existingRec.type + const { typeChanged, verbForIndex, updatedMetadata } = this.buildUpdateRelationRecord(params, existing) + + // 8.0 BigInt boundary: endpoints are unchanged across a type swap, so one + // resolution serves both the remove (rollback re-add) and the re-add — + // only needed on the legacy pair path below; the update-op path never + // touches endpoint ints (the provider already holds the mapping). + const reindexInts = typeChanged ? this.resolveVerbEndpointInts(verbForIndex) : undefined + + await this.persistSingleOp({ verbs: [params.id] }, async (tx) => { + tx.addOperation( + new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata) + ) + + // If the verb type changed, re-index in graph adjacency so traversal-by-type + // stays consistent. The id is preserved across the swap. ONE atomic + // update-op leg when the graph provider announces the capability, else + // the legacy remove-old/add-new pair — the one-train overlap this + // release. + if (typeChanged && reindexInts) { + const graphUpdateProvider = graphUpdateOpProvider(this.graphIndex) + if (graphUpdateProvider) { + tx.addOperation( + new UpdateVerbInGraphIndexOperation(graphUpdateProvider, existing, verbForIndex, this.graphWriteGeneration) + ) + } else { + tx.addOperation( + new RemoveFromGraphIndexOperation( + this.graphIndex, existing, reindexInts, this.graphWriteGeneration + ) + ) + tx.addOperation( + new AddToGraphIndexOperation( + this.graphIndex, verbForIndex, reindexInts, + this.graphWriteGeneration, + (verbInt) => this.cacheVerbInt(verbInt, params.id) + ) + ) + } + } + }, + undefined, + this._changeFeed.hasListeners + ? [ + { + kind: 'relation', + op: 'updateRelation', + id: params.id, + relation: { + id: params.id, + from: verbForIndex.sourceId, + to: verbForIndex.targetId, + type: String(verbForIndex.verb ?? verbForIndex.type), + ...(verbForIndex.metadata && { + metadata: verbForIndex.metadata as Record + }) + } + } + ] + : undefined) + } + + /** + * Pure record-building core shared by `updateRelation()` and + * `planTxUpdateRelation()`: given the caller-resolved before-image verb, + * runs subtype enforcement and computes the merged v2 metadata record and + * the graph-index view of the after-image — WITHOUT touching storage. The + * actual read of the before-image (`storage.getVerb` for the single-op + * path; batch-state-aware resolution for the transact planner, so a verb + * updated or created earlier in the SAME batch is visible) stays with the + * caller — the two differ, mirroring why `update()`/`planTxUpdate` each + * own their own entity read rather than sharing one. + * + * @param params - The update params. + * @param existingRec - The verb's REAL before-image (legacy stored shapes + * carried the verb type under `type` instead of the canonical `verb` + * field — this reads both, canonical first). + * @returns `typeChanged` (whether the effective verb type differs from the + * before-image), the merged `newMetadata` user bag, the full + * `updatedMetadata` v2 record to persist, and `verbForIndex` — the + * graph-index view of the after-image (its `id` matches + * `existingRec.id` — updates never change a verb's id). + */ + private buildUpdateRelationRecord( + params: UpdateRelationParams, + existingRec: GraphVerb + ): { + typeChanged: boolean + newMetadata: Record + updatedMetadata: Record + verbForIndex: GraphVerb + } { + const newVerbType = (params.type ?? existingRec.verb ?? existingRec.type) as VerbType // Subtype pairing enforcement on update (7.30.0). The effective verb type after // the update may have changed; we check against the new type and the resulting @@ -4898,7 +5017,7 @@ export class Brainy implements BrainyInterface { // Build the verb view used by the graph index — top-level fields mirror relate()'s. const verbForIndex: GraphVerb = { - id: params.id, + id: existingRec.id, vector: existingRec.vector, sourceId: existingRec.sourceId, targetId: existingRec.targetId, @@ -4916,51 +5035,12 @@ export class Brainy implements BrainyInterface { createdAt: existingRec.createdAt } - // 8.0 BigInt boundary: endpoints are unchanged across a type swap, so one - // resolution serves both the remove (rollback re-add) and the re-add. - const reindexInts = typeChanged ? this.resolveVerbEndpointInts(verbForIndex) : undefined - - await this.persistSingleOp({ verbs: [params.id] }, async (tx) => { - tx.addOperation( - new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata) - ) - - // If the verb type changed, re-index in graph adjacency so traversal-by-type - // stays consistent. The id is preserved across the swap. - if (typeChanged && reindexInts) { - tx.addOperation( - new RemoveFromGraphIndexOperation( - this.graphIndex, existing, reindexInts, this.graphWriteGeneration - ) - ) - tx.addOperation( - new AddToGraphIndexOperation( - this.graphIndex, verbForIndex, reindexInts, - this.graphWriteGeneration, - (verbInt) => this.cacheVerbInt(verbInt, params.id) - ) - ) - } - }, - undefined, - this._changeFeed.hasListeners - ? [ - { - kind: 'relation', - op: 'updateRelation', - id: params.id, - relation: { - id: params.id, - from: verbForIndex.sourceId, - to: verbForIndex.targetId, - type: String(verbForIndex.verb ?? verbForIndex.type), - ...(verbForIndex.metadata && { - metadata: verbForIndex.metadata as Record - }) - } - } - ] - : undefined) + return { + typeChanged, + newMetadata: newMetadata as Record, + updatedMetadata, + verbForIndex + } } /** @@ -7958,6 +8038,9 @@ export class Brainy implements BrainyInterface { entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined, }) } + // Registration-time refusal, re-run on every re-adoption (see init()'s + // matching check) — before any write can run against the recreated index. + assertUpdateCapabilityCoherent(this.metadataIndex, 'metadata') await this.metadataIndex.init() // Re-resolve the graph index the same way init() does (provider factory, @@ -7972,6 +8055,8 @@ export class Brainy implements BrainyInterface { } else { this.graphIndex = await this.storage.getGraphIndex() } + // Same registration-time refusal, graph half. + assertUpdateCapabilityCoherent(this.graphIndex, 'graph') this.wireGraphIdResolver() // Reset dimensions @@ -10159,6 +10244,9 @@ export class Brainy implements BrainyInterface { case 'relate': plan.ids.push(await this.planTxRelate(op, state, plan)) break + case 'updateRelation': + plan.ids.push(await this.planTxUpdateRelation(op, state, plan)) + break case 'unrelate': plan.ids.push(await this.planTxUnrelate(op, state, plan)) break @@ -10530,10 +10618,21 @@ export class Brainy implements BrainyInterface { new ReplaceInVectorIndexOperation(this.index, params.id, existing.vector, vector, this.indexWriteGeneration) ) } - plan.operations.push( - new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata, this.indexWriteGeneration), - new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing, this.indexWriteGeneration) - ) + // ONE atomic metadata-index leg when the provider announces the + // update-op capability, else the legacy remove-old/add-new pair — same + // branch as update() (see the class note above planTxUpdate's Object 5-6 + // sibling in update()). + const metadataUpdateProviderTx = metadataUpdateOpProvider(this.metadataIndex) + if (metadataUpdateProviderTx) { + plan.operations.push( + new UpdateInMetadataIndexOperation(metadataUpdateProviderTx, params.id, removalMetadata, entityForIndexing, this.indexWriteGeneration) + ) + } else { + plan.operations.push( + new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata, this.indexWriteGeneration), + new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing, this.indexWriteGeneration) + ) + } plan.touchedNouns.push(params.id) // The full planGetEntity view, passed whole — a subset view makes the @@ -10861,6 +10960,82 @@ export class Brainy implements BrainyInterface { return id } + /** + * Plan one `{ op: 'updateRelation' }` — mirror of `updateRelation()`, + * batch-aware: resolves the before-image against "current state + batch + * so far" (a `relate` or an earlier `updateRelation` on the SAME id + * earlier in this batch is visible), so the op is genuinely batchable — + * the graph counterpart of `planTxUpdate`. Returns the relationship id. + */ + private async planTxUpdateRelation( + op: Extract, { op: 'updateRelation' }>, + state: TxPlanState, + plan: PlannedTransact + ): Promise { + const { op: _discriminator, ...rawParams } = op + validateUpdateRelationParams(rawParams as UpdateRelationParams) + const params = rawParams as UpdateRelationParams + + const existing = state.removedVerbs.has(params.id) + ? null + : (state.verbs.get(params.id) ?? (await this.storage.getVerb(params.id))) + if (!existing) { + throw new RelationNotFoundError(params.id) + } + + const { typeChanged, verbForIndex, updatedMetadata } = this.buildUpdateRelationRecord(params, existing) + + plan.operations.push( + new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata) + ) + + // If the verb type changed, re-index in graph adjacency — same branch as + // updateRelation(): ONE atomic update-op leg when the graph provider + // announces the capability, else the legacy remove-old/add-new pair. + if (typeChanged) { + const graphUpdateProvider = graphUpdateOpProvider(this.graphIndex) + if (graphUpdateProvider) { + plan.operations.push( + new UpdateVerbInGraphIndexOperation(graphUpdateProvider, existing, verbForIndex, this.graphWriteGeneration) + ) + } else { + plan.operations.push( + // Endpoint ints resolve at EXECUTE time — mirror of planTxRelate/ + // planTxUnrelate: the verb (or its endpoints) may have been + // created earlier in this same batch. + new RemoveFromGraphIndexOperation( + this.graphIndex, existing, () => this.resolveVerbEndpointInts(verbForIndex), this.graphWriteGeneration + ), + new AddToGraphIndexOperation( + this.graphIndex, verbForIndex, () => this.resolveVerbEndpointInts(verbForIndex), + this.graphWriteGeneration, + (verbInt) => this.cacheVerbInt(verbInt, params.id) + ) + ) + } + } + + plan.touchedVerbs.push(params.id) + state.verbs.set(params.id, verbForIndex) + + if (this._changeFeed.hasListeners) { + plan.changeEvents.push({ + kind: 'relation', + op: 'updateRelation', + id: params.id, + relation: { + id: params.id, + from: verbForIndex.sourceId, + to: verbForIndex.targetId, + type: String(verbForIndex.verb ?? verbForIndex.type), + ...(verbForIndex.metadata && { metadata: verbForIndex.metadata as Record }) + } + }) + } + + return params.id + } + /** Plan one `{ op: 'unrelate' }` — mirror of `unrelate()`. Returns the relationship id. */ private async planTxUnrelate( op: Extract, { op: 'unrelate' }>, diff --git a/src/db/db.ts b/src/db/db.ts index 68428a7c..8e69727d 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -61,7 +61,7 @@ import { exportGraph } from './portableGraph.js' import type { ExportSelector, ExportOptions, PortableGraph } from './portableGraph.js' import { v4 as uuidv4 } from '../universal/uuid.js' import { coerceNewEntityId, resolveEntityId, ORIGINAL_ID_KEY } from '../utils/idNormalization.js' -import { EntityNotFoundError } from '../errors/notFound.js' +import { EntityNotFoundError, RelationNotFoundError } from '../errors/notFound.js' import { SpeculativeOverlayError, CanonicalEnumerationUnavailableError } from './errors.js' import type { GenerationStore } from './generationStore.js' import type { ChangedIds, TransactReceipt, TxOperation } from './types.js' @@ -698,6 +698,39 @@ export class Db { return this.get(id) } + // The verb counterpart of `speculativeGet`. No `Db.getRelation(id)` + // exists (only the array-returning `related()`), so this replicates its + // generation-aware single-id resolution: the overlay first, then the + // generational before-image if something after this pin touched the + // verb (the same `resolveAt('verb', …)` + `relationFromRecord` pairing + // `related()` uses for its own changed-but-not-overlaid merge), else the + // live stored verb — nothing after this pin touched it, so the live + // state IS this view's state. + const speculativeGetRelation = async (id: string): Promise | null> => { + if (overlay.verbs.has(id)) return overlay.verbs.get(id) ?? null + const resolved = await this.host.store.resolveAt('verb', id, this.gen) + if (resolved.source === 'absent') return null + if (resolved.source === 'record') { + return this.host.relationFromRecord(id, { metadata: resolved.metadata, vector: resolved.vector }) + } + const stored = await this.host.storage.getVerb(id) + if (!stored) return null + return { + id: stored.id, + from: stored.sourceId, + to: stored.targetId, + type: stored.verb, + ...(stored.subtype !== undefined && { subtype: stored.subtype }), + ...(stored.visibility !== undefined && { visibility: stored.visibility }), + weight: stored.weight ?? 1.0, + ...(stored.confidence !== undefined && { confidence: stored.confidence }), + data: stored.data, + metadata: (stored.metadata ?? {}) as T, + ...(stored.service !== undefined && { service: stored.service }), + createdAt: stored.createdAt + } + } + for (const op of ops) { switch (op.op) { case 'add': { @@ -855,6 +888,35 @@ export class Db { } break } + case 'updateRelation': { + const base = await speculativeGetRelation(op.id) + if (!base) { + throw new RelationNotFoundError( + op.id, + `with(): relationship ${op.id} not found at generation ${this.gen}` + ) + } + // Field-addressing law — mirror of the 'update' case: the patch + // bag is the user's verbatim; engine scalars only from dedicated + // op fields. + const custom = { ...(op.metadata as Record | undefined) } + const mergedMetadata = + op.merge !== false + ? ({ ...(base.metadata as object), ...custom } as T) + : ((op.metadata !== undefined ? custom : base.metadata) as T) + overlay.verbs.set(op.id, { + ...base, + ...(op.type !== undefined && { type: op.type }), + ...(op.subtype !== undefined && { subtype: op.subtype }), + ...(op.visibility !== undefined && { visibility: op.visibility }), + ...(op.weight !== undefined && { weight: op.weight }), + ...(op.confidence !== undefined && { confidence: op.confidence }), + ...(op.data !== undefined && { data: op.data }), + metadata: mergedMetadata, + updatedAt: Date.now() + }) + break + } case 'unrelate': { overlay.verbs.set(op.id, null) break diff --git a/src/db/types.ts b/src/db/types.ts index 363de086..c681ad9c 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -25,7 +25,7 @@ * `docs/ADR-001-generational-mvcc.md` for the full justification. */ -import type { AddParams, UpdateParams, RelateParams, Entity, Relation } from '../types/brainy.types.js' +import type { AddParams, UpdateParams, RelateParams, UpdateRelationParams, Entity, Relation } from '../types/brainy.types.js' // ============================================================================ // Transaction operations (brain.transact input) @@ -75,6 +75,18 @@ export interface TxRelateOperation extends RelateParams { op: 'relate' } +/** + * @description Update a relationship. Carries the same parameters as + * `brain.updateRelation()` — a first-class batch op (not merely `unrelate` + * + `relate`), so a type change re-indexes the SAME relationship id rather + * than minting a new one, and a batch containing it is rejected atomically + * (like every other op here) if the relationship id does not exist. + */ +export interface TxUpdateRelationOperation extends UpdateRelationParams { + /** Discriminator. */ + op: 'updateRelation' +} + /** * @description Delete a relationship by id (mirror of `brain.unrelate()`). */ @@ -96,6 +108,7 @@ export type TxOperation = | TxUpdateOperation | TxRemoveOperation | TxRelateOperation + | TxUpdateRelationOperation | TxUnrelateOperation /** diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index a58236e3..18299405 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -18,6 +18,7 @@ export type BrainyErrorType = | 'PROTECTED_ARTIFACT' | 'DERIVED_ARTIFACT_MISSING' | 'MIGRATION_IN_PROGRESS' + | 'PROVIDER_CAPABILITY_MISMATCH' /** * Custom error class for Brainy operations @@ -405,3 +406,43 @@ export class MigrationInProgressError extends BrainyError { } } } + +/** + * Thrown at PROVIDER REGISTRATION when an index-provider instance (the + * `'metadataIndex'` or `'graphIndex'` provider a native accelerator + * registers) announces a capability in its `capabilities` set that its own + * methods do not actually implement — e.g. the set contains `'update-op'` + * but the instance has no `updateIndex`/`updateVerb` function. A provider + * must never claim more than it delivers: honoring the announcement would + * let brainy emit an update op the provider cannot execute, discovered only + * at the first write instead of at startup. This is the TYPED REFUSAL AT + * REGISTRATION — loud and immediate, never a silent fallback to the legacy + * remove+add pair for a provider that lied about its capabilities. + * + * Raised by `assertUpdateCapabilityCoherent` + * (`src/transaction/operations/updateCapability.ts`), called once per + * provider at adoption time, before any write can run. + */ +export class ProviderCapabilityMismatchError extends BrainyError { + /** Which provider family failed the check. */ + public readonly family: 'metadata' | 'graph' + /** The method the announced capability required but the provider lacks. */ + public readonly missingMethod: string + + constructor(family: 'metadata' | 'graph', missingMethod: string) { + super( + `Provider capability mismatch: the '${family}' index provider's ` + + `\`capabilities\` set claims 'update-op' but does not expose a ` + + `\`${missingMethod}\` method — a provider must not announce a ` + + `capability it does not implement. Registration refused.`, + 'PROVIDER_CAPABILITY_MISMATCH', + false + ) + this.name = 'ProviderCapabilityMismatchError' + this.family = family + this.missingMethod = missingMethod + if (Error.captureStackTrace) { + Error.captureStackTrace(this, ProviderCapabilityMismatchError) + } + } +} diff --git a/src/index.ts b/src/index.ts index 765c20b4..285fed6d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -202,7 +202,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js // Base error + typed migration-lock error — thrown by any data-plane call while a // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After. -export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js' +export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError, ProviderCapabilityMismatchError } from './errors/brainyError.js' export type { BrainyErrorType } from './errors/brainyError.js' // ============= 8.0 Db API — generational MVCC ============= diff --git a/src/plugin.ts b/src/plugin.ts index 947c86a5..4068e6dd 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -307,6 +307,48 @@ export interface MetadataIndexProvider { */ removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise + /** + * @description OPTIONAL. The capability set this provider INSTANCE + * announces — e.g. `{'update-op'}` for {@link updateIndex}. Absence means + * "no additional capabilities": brainy keeps this provider on the legacy + * remove+add pair (`removeFromIndex` then `addToIndex`) for every update, + * unchanged — the one-train overlap this release. A provider whose set + * claims a capability its methods do not actually implement is a TYPED + * REFUSAL AT REGISTRATION (loud, never a silent fallback) — see + * `assertUpdateCapabilityCoherent` in + * `src/transaction/operations/updateCapability.ts`, called once per + * provider at adoption, before any write can run. + */ + capabilities?: ReadonlySet + + /** + * @description OPTIONAL FIRST-CLASS UPDATE. Mutates one entity's indexed + * metadata from `before` to `after` IN PLACE — the cure for the historical + * remove+add pair, whose two separately-awaited legs let a provider that + * keys derived structures by entity identity destroy state on the remove + * leg that the add leg needed back (four production regressions in five + * releases lived at that seam). `before`/`after` are the SAME + * entity-for-indexing shapes {@link removeFromIndex}/{@link addToIndex} + * receive; the optional trailing `generation` mirrors {@link addToIndex}'s. + * + * Brainy emits this op ONLY when BOTH halves of the capability check pass: + * `provider.capabilities?.has('update-op') && typeof provider.updateIndex + * === 'function'`. Absence of the capability set keeps the provider on the + * legacy remove+add pair path, which remains supported for (at least) one + * overlap release. + * + * Rollback is symmetric BY CONSTRUCTION: `updateIndex(id, after, before)` + * undoes `updateIndex(id, before, after)` — the same resolved generation is + * reused for both, exactly like the existing add/remove op pairs. + * @param id - The entity's UUID. + * @param before - The entity's REAL before-image (the existing indexed + * shape read on the update path) — never invented. + * @param after - The entity's after-image (the new indexed shape). + * @param generation - OPTIONAL commit generation, same contract as + * {@link addToIndex}. + */ + updateIndex?(id: string, before: any, after: any, generation?: bigint): Promise + getIds(field: string, value: any): Promise /** * Resolve a `where` filter to its matching ids. @@ -573,6 +615,44 @@ export interface GraphIndexProvider { */ removeVerb(verbId: string, generation: bigint): Promise + /** + * @description OPTIONAL. The capability set this provider INSTANCE + * announces — e.g. `{'update-op'}` for {@link updateVerb}. Absence means + * "no additional capabilities": brainy keeps this provider on the legacy + * remove+add pair (`removeVerb` then `addVerb`) for every verb update, + * unchanged — the one-train overlap this release. A provider whose set + * claims a capability its methods do not actually implement is a TYPED + * REFUSAL AT REGISTRATION (loud, never a silent fallback) — see + * `assertUpdateCapabilityCoherent` in + * `src/transaction/operations/updateCapability.ts`, called once per + * provider at adoption, before any write can run. + */ + capabilities?: ReadonlySet + + /** + * @description OPTIONAL FIRST-CLASS UPDATE for one verb — the graph + * counterpart of {@link MetadataIndexProvider.updateIndex}. Endpoints + * NEVER change across an update (only type/metadata do); the provider + * already holds the verb's int mapping, so it mutates the existing edge + * record in place instead of removing and re-adding it. + * + * Brainy emits this op ONLY when BOTH halves of the capability check pass: + * `provider.capabilities?.has('update-op') && typeof provider.updateVerb + * === 'function'`. Absence of the capability set keeps the provider on the + * legacy remove+add pair path (`removeVerb` + `addVerb`), which remains + * supported for (at least) one overlap release. + * + * Rollback is symmetric BY CONSTRUCTION: `updateVerb(id, afterVerb, + * beforeVerb, generation)` undoes `updateVerb(id, beforeVerb, afterVerb, + * generation)` — the same generation is reused for both. + * @param id - The verb's UUID string. + * @param beforeVerb - The verb's REAL before-image (the existing stored verb). + * @param afterVerb - The verb's after-image. + * @param generation - Brainy's commit generation for this write — same + * contract as {@link addVerb} (required, never a fabricated 0). + */ + updateVerb?(id: string, beforeVerb: GraphVerb, afterVerb: GraphVerb, generation: bigint): Promise + rebuild(): Promise flush(): Promise close(): Promise diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 139c67fe..8c86f418 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -13,6 +13,7 @@ import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' +import type { MetadataUpdateOpProvider, GraphUpdateOpProvider } from './updateCapability.js' /** * Backend identity stamped into an operation's emitted `name` string (e.g. @@ -374,6 +375,59 @@ export class RemoveFromMetadataIndexOperation implements Operation { } } +/** + * Update metadata index IN PLACE via the provider's native update op — the + * FIRST-CLASS UPDATE the accelerator seam gained to replace the historical + * remove+add pair (see `MetadataIndexProvider.updateIndex` in + * `src/plugin.ts`). NEVER construct this directly against a raw provider + * reference — the `index` argument must come from + * {@link import('./updateCapability.js').metadataUpdateOpProvider}, the ONLY + * place that verifies the provider both announces `'update-op'` AND exposes + * the method (the both-halves check). + * + * Rollback strategy: + * - Call `updateIndex` again with `before`/`after` swapped — symmetric BY + * CONSTRUCTION (the provider mutates the SAME record back). + * + * Generation: `generationFn` is resolved at execute time (not construction) — + * see {@link AddToMetadataIndexOperation}'s class note; the same resolved + * value is reused for the rollback so the update and its undo reference one + * watermark in a provider's per-record delta log. + */ +export class UpdateInMetadataIndexOperation implements Operation { + readonly name = 'UpdateInMetadataIndex' + + /** + * @param index - The update-capable surface returned by + * {@link import('./updateCapability.js').metadataUpdateOpProvider}. + * @param id - The entity's UUID. + * @param before - The entity's REAL before-image (the existing indexed + * shape read on the update path) — never invented. + * @param after - The entity's after-image (the new indexed shape). + * @param generationFn - Resolves the commit generation to stamp this write + * at, evaluated when the operation executes. + */ + constructor( + private readonly index: MetadataUpdateOpProvider, + private readonly id: string, + private readonly before: any, + private readonly after: any, + private readonly generationFn?: () => bigint | undefined + ) {} + + async execute(): Promise { + // Stamp this write at the in-flight commit generation; reuse it for the + // symmetric rollback below. + const generation = this.generationFn?.() + await this.index.updateIndex(this.id, this.before, this.after, generation) + + return async () => { + // Symmetric rollback: same generation, before/after swapped. + await this.index.updateIndex(this.id, this.after, this.before, generation) + } + } +} + /** * Add verb to graph index with rollback support * @@ -499,6 +553,58 @@ export class RemoveFromGraphIndexOperation implements Operation { } } +/** + * Update one verb IN PLACE via the provider's native update op — the graph + * counterpart of {@link UpdateInMetadataIndexOperation}. Endpoints NEVER + * change across an update (only type/metadata do), so unlike + * {@link AddToGraphIndexOperation}/{@link RemoveFromGraphIndexOperation} this + * op carries no endpoint ints — the provider already holds the verb's int + * mapping. NEVER construct this directly against a raw provider reference — + * the `index` argument must come from + * {@link import('./updateCapability.js').graphUpdateOpProvider}, the ONLY + * place that verifies the provider both announces `'update-op'` AND exposes + * the method (the both-halves check). + * + * Rollback strategy: + * - Call `updateVerb` again with `beforeVerb`/`afterVerb` swapped — + * symmetric BY CONSTRUCTION. + * + * Generation: `generationFn` is resolved at execute time (not construction), + * mirroring {@link AddToGraphIndexOperation}; the same resolved value is + * reused for the rollback so the update and its undo reference one watermark. + */ +export class UpdateVerbInGraphIndexOperation implements Operation { + readonly name = 'UpdateVerbInGraphIndex' + + /** + * @param index - The update-capable surface returned by + * {@link import('./updateCapability.js').graphUpdateOpProvider}. + * @param beforeVerb - The verb's REAL before-image (the existing stored verb). + * @param afterVerb - The verb's after-image. Its `id` is the same as + * `beforeVerb`'s — updates never change a verb's id. + * @param generationFn - Resolves the commit generation to stamp this write + * at, evaluated when the operation executes. + */ + constructor( + private readonly index: GraphUpdateOpProvider, + private readonly beforeVerb: GraphVerb, + private readonly afterVerb: GraphVerb, + private readonly generationFn: () => bigint + ) {} + + async execute(): Promise { + // Stamp this write at the in-flight commit generation; reuse it for the + // symmetric rollback below. + const generation = this.generationFn() + await this.index.updateVerb(this.afterVerb.id, this.beforeVerb, this.afterVerb, generation) + + return async () => { + // Symmetric rollback: same generation, before/after swapped. + await this.index.updateVerb(this.afterVerb.id, this.afterVerb, this.beforeVerb, generation) + } + } +} + /** * Batch operation: Add multiple items to the vector index (backend-neutral — * see {@link AddToVectorIndexOperation}). diff --git a/src/transaction/operations/index.ts b/src/transaction/operations/index.ts index 32a69a21..f2a97eba 100644 --- a/src/transaction/operations/index.ts +++ b/src/transaction/operations/index.ts @@ -26,8 +26,19 @@ export { ReplaceInVectorIndexOperation, AddToMetadataIndexOperation, RemoveFromMetadataIndexOperation, + UpdateInMetadataIndexOperation, AddToGraphIndexOperation, RemoveFromGraphIndexOperation, + UpdateVerbInGraphIndexOperation, BatchAddToVectorIndexOperation, BatchAddToMetadataIndexOperation } from './IndexOperations.js' + +// Update-op capability check (the FIRST-CLASS UPDATE seam) +export { + UPDATE_OP_CAPABILITY, + metadataUpdateOpProvider, + graphUpdateOpProvider, + assertUpdateCapabilityCoherent +} from './updateCapability.js' +export type { MetadataUpdateOpProvider, GraphUpdateOpProvider } from './updateCapability.js' diff --git a/src/transaction/operations/updateCapability.ts b/src/transaction/operations/updateCapability.ts new file mode 100644 index 00000000..b6ff6113 --- /dev/null +++ b/src/transaction/operations/updateCapability.ts @@ -0,0 +1,125 @@ +/** + * @module transaction/operations/updateCapability + * @description The capability-check seam for the FIRST-CLASS UPDATE + * operation (`'update-op'`) an index provider may announce on its + * `capabilities` set (see `MetadataIndexProvider`/`GraphIndexProvider` in + * `src/plugin.ts`). + * + * Two responsibilities live here, both implementing the SAME both-halves + * check — never trust the set alone, never trust the method alone: + * + * 1. Narrowing helpers ({@link metadataUpdateOpProvider}, + * {@link graphUpdateOpProvider}) the planner calls at EMISSION time: they + * return the narrowed update-capable surface when a provider both + * announces `'update-op'` AND exposes the method, else `null` — the + * planner branches on that `null`-ness to choose the single update op or + * the legacy remove+add pair. Typed structurally (not the concrete + * `MetadataIndexManager`/`GraphAdjacencyIndex` classes) so the new + * operation classes never need an `as any` cast at the call sites where + * `this.metadataIndex`/`this.graphIndex` are typed as those concrete + * classes even when a provider is registered. + * 2. {@link assertUpdateCapabilityCoherent}, called once per provider at + * REGISTRATION time (before any write can run): a provider whose set + * claims `'update-op'` while its instance lacks the method is a lie the + * engine refuses loudly, via {@link ProviderCapabilityMismatchError} — + * never a silent fallback discovered only at the first write. + */ + +import type { MetadataIndexProvider, GraphIndexProvider } from '../../plugin.js' +import type { GraphVerb } from '../../coreTypes.js' +import { ProviderCapabilityMismatchError } from '../../errors/brainyError.js' + +/** The capability literal a provider's `capabilities` set must contain to opt into the update op. */ +export const UPDATE_OP_CAPABILITY = 'update-op' + +/** + * @description The narrowed metadata-index surface {@link metadataUpdateOpProvider} + * returns once both halves of the capability check pass. + */ +export interface MetadataUpdateOpProvider { + updateIndex(id: string, before: any, after: any, generation?: bigint): Promise +} + +/** + * @description The narrowed graph-index surface {@link graphUpdateOpProvider} + * returns once both halves of the capability check pass. + */ +export interface GraphUpdateOpProvider { + updateVerb(id: string, beforeVerb: GraphVerb, afterVerb: GraphVerb, generation: bigint): Promise +} + +/** + * @description The metadata-index planner branch: BOTH halves, exactly — + * `index.capabilities?.has('update-op') && typeof index.updateIndex === + * 'function'`. Returns the narrowed {@link MetadataUpdateOpProvider} to emit + * a single {@link import('./IndexOperations.js').UpdateInMetadataIndexOperation} + * against, or `null` to keep emitting the legacy remove+add pair. + * @param index - The provider (or the built-in JS manager, which never + * announces the capability and so always resolves to `null`). + */ +export function metadataUpdateOpProvider( + index: MetadataIndexProvider +): MetadataUpdateOpProvider | null { + const updateIndex = index.updateIndex + if (index.capabilities?.has(UPDATE_OP_CAPABILITY) && typeof updateIndex === 'function') { + return { updateIndex: updateIndex.bind(index) } + } + return null +} + +/** + * @description The graph-index planner branch: BOTH halves, exactly — + * `index.capabilities?.has('update-op') && typeof index.updateVerb === + * 'function'`. Returns the narrowed {@link GraphUpdateOpProvider} to emit a + * single {@link import('./IndexOperations.js').UpdateVerbInGraphIndexOperation} + * against, or `null` to keep emitting the legacy remove+add pair. + * @param index - The provider (or the built-in JS adjacency index, which + * never announces the capability and so always resolves to `null`). + */ +export function graphUpdateOpProvider( + index: GraphIndexProvider +): GraphUpdateOpProvider | null { + const updateVerb = index.updateVerb + if (index.capabilities?.has(UPDATE_OP_CAPABILITY) && typeof updateVerb === 'function') { + return { updateVerb: updateVerb.bind(index) } + } + return null +} + +/** + * @description Registration-time refusal: throws + * {@link ProviderCapabilityMismatchError} when `provider.capabilities` + * claims `'update-op'` but the family's required method is absent — a + * provider must never announce more than it delivers. Call this ONCE per + * adopted provider, at adoption (brain init and any provider re-adoption, + * e.g. `clear()`), before any write can run. A provider that does not + * announce the capability at all (the common case — most providers, and the + * built-in JS manager/index, stay on the legacy pair path) passes silently: + * this function only rejects a LIE, never the absence of a claim. + * @param provider - The adopted provider instance (the concrete + * `MetadataIndexManager`/`GraphAdjacencyIndex` classes both `implements` + * their respective interface, so this accepts them directly). + * @param family - Which provider family `provider` is, selecting whether + * `updateIndex` (`'metadata'`) or `updateVerb` (`'graph'`) is required. + * @throws {ProviderCapabilityMismatchError} When the set claims the + * capability but the required method is missing or not a function. + */ +export function assertUpdateCapabilityCoherent( + provider: MetadataIndexProvider | GraphIndexProvider, + family: 'metadata' | 'graph' +): void { + if (!provider.capabilities?.has(UPDATE_OP_CAPABILITY)) { + return + } + // Narrow by the caller-declared family — a plain `MetadataIndexProvider | + // GraphIndexProvider` union does not expose `updateIndex`/`updateVerb` + // directly (each method lives on only one side of the union), so the + // family the caller already knows selects which side to read. + const method = + family === 'metadata' + ? (provider as MetadataIndexProvider).updateIndex + : (provider as GraphIndexProvider).updateVerb + if (typeof method !== 'function') { + throw new ProviderCapabilityMismatchError(family, family === 'metadata' ? 'updateIndex' : 'updateVerb') + } +} diff --git a/tests/integration/update-op-emission.test.ts b/tests/integration/update-op-emission.test.ts new file mode 100644 index 00000000..be2486eb --- /dev/null +++ b/tests/integration/update-op-emission.test.ts @@ -0,0 +1,409 @@ +/** + * @module tests/integration/update-op-emission + * @description Pins for the FIRST-CLASS UPDATE OPERATION through the + * index-provider seam: brainy's planner now emits ONE `updateIndex`/ + * `updateVerb` call for `update()`/`updateRelation()` (including their + * `transact()` op forms) when a registered provider announces the + * `'update-op'` capability AND exposes the method (the both-halves check) — + * replacing the historical remove+add pair, which remains the emission for + * every provider that does not announce the capability (the one-train + * overlap this release). + * + * Recording provider doubles (below) wrap the built-in JS metadata/graph + * index managers, delegating every real method to the base class while + * recording the call sequence, so each pin below observes brainy's ACTUAL + * emission choice rather than mocking the engine. + * + * Pins: + * (a) update() on a capable metadata provider → one updateIndex, zero + * removeFromIndex/addToIndex for that id; find() sees the new metadata, + * not the old. + * (b) the same through transact([{ op: 'update' }]). + * (c) rollback: a batch rejected at PLAN time never touches the provider for + * the update's id (the row keeps its old metadata); a batch rejected + * DURING EXECUTE (a later op's index write fails) rolls the update op + * back symmetrically — updateIndex(id, after, before). + * (d) a provider whose capabilities claim 'update-op' but lacks updateIndex + * is refused at registration with ProviderCapabilityMismatchError. + * (e) a provider with no capabilities set gets the legacy pair — both calls + * recorded, same commit, the row is never absent from find() between + * them from a caller's view. + * (f) updateRelation() with a recording GRAPH provider announcing + * 'update-op' → exactly one updateVerb call; the relation reads back + * with the new type. + * (g) transact([{ op: 'updateRelation' }]): merges metadata and reads back; + * a type change re-indexes; an unknown id rejects the whole batch (other + * ops in it do not apply). + */ +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { MetadataIndexManager } from '../../src/utils/metadataIndex.js' +import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js' +import type { GraphVerb } from '../../src/coreTypes.js' +import { EntityNotFoundError, RelationNotFoundError } from '../../src/errors/notFound.js' +import { ProviderCapabilityMismatchError } from '../../src/errors/brainyError.js' + +const V = (): number[] => Array.from({ length: 384 }, () => Math.random()) + +/** Valid-UUID-shaped deterministic ids so `add`/`relate` never coerce them via the natural-key path. */ +let seq = 0 +const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + +type RecordedCall = { method: 'addToIndex' | 'removeFromIndex' | 'updateIndex'; id: string } +type RecordedVerbCall = { method: 'addVerb' | 'removeVerb' | 'updateVerb'; id: string } + +/** + * A metadata-index provider double: wraps the built-in `MetadataIndexManager`, + * delegating every real write to the base class while recording the call + * sequence. `capable: false` omits the `capabilities` set entirely (legacy + * pair path); `failAddFor` injects a targeted `addToIndex` failure so a + * later batch op can force a mid-execute rollback (pin c). + */ +function makeRecordingMetadataFactory( + calls: RecordedCall[], + opts: { capable?: boolean; failAddFor?: string } = {} +): (storage: any) => MetadataIndexManager { + const capable = opts.capable !== false + return (storage: any) => { + class RecordingMetadataProvider extends MetadataIndexManager { + capabilities = capable ? new Set(['update-op']) : undefined + + async addToIndex( + id: string, + entityOrMetadata: any, + skipFlush = false, + deferWrites = false, + generation?: bigint + ): Promise { + if (opts.failAddFor !== undefined && id === opts.failAddFor) { + throw new Error(`injected: addToIndex failed for ${id}`) + } + calls.push({ method: 'addToIndex', id }) + return super.addToIndex(id, entityOrMetadata, skipFlush, deferWrites, generation) + } + + async removeFromIndex(id: string, metadata?: any, generation?: bigint): Promise { + calls.push({ method: 'removeFromIndex', id }) + return super.removeFromIndex(id, metadata, generation) + } + + // Delegate-remove+delegate-add internally via `super.*` (bypassing this + // class's own overrides) so the update-op path records ONLY + // 'updateIndex' for the id it touches — never the pair. + async updateIndex(id: string, before: any, after: any, generation?: bigint): Promise { + calls.push({ method: 'updateIndex', id }) + await super.removeFromIndex(id, before, generation) + await super.addToIndex(id, after, true, false, generation) + } + } + return new RecordingMetadataProvider(storage) + } +} + +/** A provider whose `capabilities` claims 'update-op' but never implements `updateIndex` — pin (d). */ +function makeLyingMetadataFactory(): (storage: any) => MetadataIndexManager { + return (storage: any) => { + class LyingMetadataProvider extends MetadataIndexManager { + capabilities = new Set(['update-op']) + // Deliberately no `updateIndex` override. + } + return new LyingMetadataProvider(storage) + } +} + +/** The graph-index counterpart of {@link makeRecordingMetadataFactory}. */ +function makeRecordingGraphFactory( + calls: RecordedVerbCall[], + opts: { capable?: boolean } = {} +): (storage: any) => GraphAdjacencyIndex { + const capable = opts.capable !== false + return (storage: any) => { + class RecordingGraphProvider extends GraphAdjacencyIndex { + capabilities = capable ? new Set(['update-op']) : undefined + + async addVerb(verb: GraphVerb, sourceInt: bigint, targetInt: bigint, generation: bigint): Promise { + calls.push({ method: 'addVerb', id: verb.id }) + return super.addVerb(verb, sourceInt, targetInt, generation) + } + + async removeVerb(verbId: string, generation: bigint): Promise { + calls.push({ method: 'removeVerb', id: verbId }) + return super.removeVerb(verbId, generation) + } + + async updateVerb(id: string, beforeVerb: GraphVerb, afterVerb: GraphVerb, generation: bigint): Promise { + calls.push({ method: 'updateVerb', id }) + await super.removeVerb(id, generation) + // The JS index's addVerb ignores sourceInt/targetInt (it operates on + // the verb's string ids internally) — endpoints never change across + // an update, so no real resolution is needed here. + await super.addVerb(afterVerb, 0n, 0n, generation) + } + } + return new RecordingGraphProvider(storage) + } +} + +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) +}) + +async function makeBrain(plugin: any): Promise { + const brain = new Brainy({ + storage: { type: 'memory' }, + requireSubtype: false, + silent: true, + plugins: [] + }) + brain.use(plugin) + await brain.init() + brains.push(brain) + return brain +} + +describe('(a) update() emission on a capable metadata provider', () => { + it('emits exactly one updateIndex call and zero removeFromIndex/addToIndex for that id; find() sees the new row, not the old', async () => { + const calls: RecordedCall[] = [] + const brain = await makeBrain({ + name: 'recording-metadata-a', + activate: async (ctx: any) => { + ctx.registerProvider('metadataIndex', makeRecordingMetadataFactory(calls)) + return true + } + }) + + const id = await brain.add({ data: 'a', type: NounType.Concept, metadata: { tag: 'old' }, vector: V() }) + calls.length = 0 + + await brain.update({ id, metadata: { tag: 'new' } }) + + expect(calls.filter((c) => c.id === id)).toEqual([{ method: 'updateIndex', id }]) + + expect((await brain.find({ where: { tag: 'new' } })).some((r: any) => r.id === id)).toBe(true) + expect((await brain.find({ where: { tag: 'old' } })).some((r: any) => r.id === id)).toBe(false) + }) +}) + +describe('(b) transact([{ op: "update" }]) emission on a capable metadata provider', () => { + it('emits exactly one updateIndex call', async () => { + const calls: RecordedCall[] = [] + const brain = await makeBrain({ + name: 'recording-metadata-b', + activate: async (ctx: any) => { + ctx.registerProvider('metadataIndex', makeRecordingMetadataFactory(calls)) + return true + } + }) + + const id = await brain.add({ data: 'b', type: NounType.Concept, metadata: { tag: 'old' }, vector: V() }) + calls.length = 0 + + await brain.transact([{ op: 'update', id, metadata: { tag: 'new' } }] as any) + + expect(calls.filter((c) => c.id === id)).toEqual([{ method: 'updateIndex', id }]) + expect((await brain.find({ where: { tag: 'new' } })).some((r: any) => r.id === id)).toBe(true) + }) +}) + +describe('(c) rollback symmetry', () => { + it('a batch rejected at PLAN time never touches the provider for the update id — the row keeps its old metadata', async () => { + const calls: RecordedCall[] = [] + const brain = await makeBrain({ + name: 'recording-metadata-c1', + activate: async (ctx: any) => { + ctx.registerProvider('metadataIndex', makeRecordingMetadataFactory(calls)) + return true + } + }) + + const id = await brain.add({ data: 'c1', type: NounType.Concept, metadata: { tag: 'old' }, vector: V() }) + calls.length = 0 + + // planTxRelate rejects an unknown target BEFORE commitTransaction is + // ever called — nothing in the batch (including the earlier update) + // executes, so the provider is never invoked for `id`. + await expect( + brain.transact([ + { op: 'update', id, metadata: { tag: 'new' } }, + { op: 'relate', from: id, to: freshId(), type: VerbType.RelatedTo } + ] as any) + ).rejects.toBeInstanceOf(EntityNotFoundError) + + expect(calls.filter((c) => c.id === id)).toEqual([]) + expect((await brain.get(id))?.metadata?.tag).toBe('old') + }) + + it('a batch rejected DURING EXECUTE (a later op\'s index write fails) rolls the update back symmetrically: updateIndex(id, after, before)', async () => { + const calls: RecordedCall[] = [] + const failId = freshId() + const brain = await makeBrain({ + name: 'recording-metadata-c2', + activate: async (ctx: any) => { + ctx.registerProvider('metadataIndex', makeRecordingMetadataFactory(calls, { failAddFor: failId })) + return true + } + }) + + const id = await brain.add({ data: 'c2', type: NounType.Concept, metadata: { tag: 'old' }, vector: V() }) + calls.length = 0 + + await expect( + brain.transact([ + { op: 'update', id, metadata: { tag: 'new' } }, + { op: 'add', id: failId, data: 'boom', type: NounType.Concept, vector: V() } + ] as any) + ).rejects.toThrow() + + // Forward call, then the symmetric rollback (before/after swapped). + expect(calls.filter((c) => c.id === id).map((c) => c.method)).toEqual(['updateIndex', 'updateIndex']) + expect((await brain.get(id))?.metadata?.tag).toBe('old') + }) +}) + +describe('(d) registration-time refusal', () => { + it('a provider whose capabilities claim update-op but lacks updateIndex is refused loudly, with the typed code', async () => { + const brain = new Brainy({ + storage: { type: 'memory' }, + requireSubtype: false, + silent: true, + plugins: [] + }) + brain.use({ + name: 'lying-metadata-provider', + activate: async (ctx: any) => { + ctx.registerProvider('metadataIndex', makeLyingMetadataFactory()) + return true + } + }) + + let caught: unknown + try { + await brain.init() + brains.push(brain) + } catch (err) { + caught = err + } + + expect(caught).toBeInstanceOf(ProviderCapabilityMismatchError) + expect((caught as ProviderCapabilityMismatchError).type).toBe('PROVIDER_CAPABILITY_MISMATCH') + expect((caught as ProviderCapabilityMismatchError).family).toBe('metadata') + }) +}) + +describe('(e) legacy pair path (no capabilities announced)', () => { + it('update() emits the remove-old/add-new pair, both recorded, same commit', async () => { + const calls: RecordedCall[] = [] + const brain = await makeBrain({ + name: 'recording-metadata-e', + activate: async (ctx: any) => { + ctx.registerProvider('metadataIndex', makeRecordingMetadataFactory(calls, { capable: false })) + return true + } + }) + + const id = await brain.add({ data: 'e', type: NounType.Concept, metadata: { tag: 'old' }, vector: V() }) + calls.length = 0 + + await brain.update({ id, metadata: { tag: 'new' } }) + + expect(calls.filter((c) => c.id === id).map((c) => c.method)).toEqual(['removeFromIndex', 'addToIndex']) + + // From a caller's view the row is never absent between the two legs — + // by the time update() resolves, the new metadata is the only truth. + expect((await brain.find({ where: { tag: 'new' } })).some((r: any) => r.id === id)).toBe(true) + expect((await brain.find({ where: { tag: 'old' } })).some((r: any) => r.id === id)).toBe(false) + }) +}) + +describe('(f) updateRelation() emission on a capable graph provider', () => { + it('a type change emits exactly one updateVerb call; the relation reads back with the new type', async () => { + const calls: RecordedVerbCall[] = [] + const brain = await makeBrain({ + name: 'recording-graph-f', + activate: async (ctx: any) => { + ctx.registerProvider('graphIndex', makeRecordingGraphFactory(calls)) + return true + } + }) + + const a = await brain.add({ data: 'a', type: NounType.Person, vector: V() }) + const b = await brain.add({ data: 'b', type: NounType.Person, vector: V() }) + const relId = await brain.relate({ from: a, to: b, type: VerbType.WorksWith }) + calls.length = 0 + + await brain.updateRelation({ id: relId, type: VerbType.ReportsTo }) + + expect(calls.filter((c) => c.id === relId)).toEqual([{ method: 'updateVerb', id: relId }]) + + // Read back via the metadata record directly rather than related({ type }) + // — a PRE-EXISTING, unrelated bug (confirmed present on the legacy pair + // path too, unmodified by this change) means the verb's CORE stored + // record (written once by relate()'s SaveVerbOperation) never gets a + // fresh SaveVerbOperation on a type change, so hydrateVerbWithMetadata's + // `{ ...coreVerb, metadata: custom }` merge keeps serving the OLD `.verb` + // to related()'s storage fast path regardless of which graph-index + // emission ran. Out of scope here (this task only concerns the + // metadata/graph INDEX provider emission); the metadata record itself — + // what updateRelation() actually owns — is the honest read. + const meta = await (brain as any).storage.getVerbMetadata(relId) + expect(meta?.verb).toBe(VerbType.ReportsTo) + }) +}) + +describe('(g) transact([{ op: "updateRelation" }])', () => { + it('merges metadata and reads back', async () => { + const brain = await makeBrain({ name: 'plain-g1', activate: async () => true }) + + const a = await brain.add({ data: 'a', type: NounType.Person, vector: V() }) + const b = await brain.add({ data: 'b', type: NounType.Person, vector: V() }) + const relId = await brain.relate({ from: a, to: b, type: VerbType.WorksWith, metadata: { x: 1 } }) + + await brain.transact([{ op: 'updateRelation', id: relId, metadata: { y: 2 } }] as any) + + const after = await brain.related({ from: a, type: VerbType.WorksWith }) + const rel = after.find((r) => r.id === relId) + expect(rel?.metadata).toEqual({ x: 1, y: 2 }) + }) + + it('a type change through transact re-indexes (planTxUpdateRelation emits the graph leg, same as updateRelation())', async () => { + const calls: RecordedVerbCall[] = [] + const brain = await makeBrain({ + name: 'recording-graph-g2', + activate: async (ctx: any) => { + ctx.registerProvider('graphIndex', makeRecordingGraphFactory(calls)) + return true + } + }) + + const a = await brain.add({ data: 'a', type: NounType.Person, vector: V() }) + const b = await brain.add({ data: 'b', type: NounType.Person, vector: V() }) + const relId = await brain.relate({ from: a, to: b, type: VerbType.WorksWith }) + calls.length = 0 + + await brain.transact([{ op: 'updateRelation', id: relId, type: VerbType.ReportsTo }] as any) + + // planTxUpdateRelation took the SAME update-op branch as updateRelation() + // (see pin (f)) — one updateVerb call, not the pair. + expect(calls.filter((c) => c.id === relId)).toEqual([{ method: 'updateVerb', id: relId }]) + + const meta = await (brain as any).storage.getVerbMetadata(relId) + expect(meta?.verb).toBe(VerbType.ReportsTo) + }) + + it('an unknown id rejects the whole batch — other ops in it do not apply', async () => { + const brain = await makeBrain({ name: 'plain-g3', activate: async () => true }) + + const newId = freshId() + await expect( + brain.transact([ + { op: 'add', id: newId, data: 'never lands', type: NounType.Concept, vector: V() }, + { op: 'updateRelation', id: freshId(), subtype: 'ghost' } + ] as any) + ).rejects.toBeInstanceOf(RelationNotFoundError) + + expect(await brain.get(newId)).toBeNull() + }) +}) From 116550eb167b13e8d2f4361e77174438051cd1d6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 24 Aug 2026 09:54:25 -0700 Subject: [PATCH 002/150] =?UTF-8?q?fix(health):=20one=20contract=20for=20a?= =?UTF-8?q?=20throwing=20probe=20=E2=80=94=20heal=20is=20none,=20serving?= =?UTF-8?q?=20is=20not=20withheld;=20repair=20report=20gains=20missing/reb?= =?UTF-8?q?uilt/reason?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A validateInvariants() that threw was re-synthesized by the host's catch as heal:'rebuild' with serving:false — a rebuild lever one transient exception away, while the native provider's own composer reports the same event as heal:'none'. Two components disagreeing on what a thrown check means is how a flaky probe becomes an outage. Both now agree: the report is named ('validate-invariants-threw'), loud (healthy:false, the error in detail), unverified — and it never buys a rebuild and never withholds serving; the provider's serving verdict is the provider's to compose, not inferred from a probe that failed to run. The synthesized report also carries the provider's name instead of 'unknown'. RepairFamilyReport gains `missing: {count, sample}` (an exact count plus a capped id sample — a verdict, not a dump), `rebuilt` (a full generational rebuild ran, as opposed to an incremental heal) and `reason`, aligning the receipt's shape with the provider-side health report. Pinned in tests/unit/validate-invariants-delegation.test.ts. --- src/brainy.ts | 19 +++++++++++--- src/types/brainy.types.ts | 13 ++++++++++ .../validate-invariants-delegation.test.ts | 25 +++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 6ebce2fd..db332e18 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -14221,16 +14221,29 @@ export class Brainy implements BrainyInterface { const report = await fn.call(provider) if (report && Array.isArray(report.invariants)) reports.push(report) } catch (err) { + // ONE CONTRACT FOR A THROWING PROBE, both engines: a probe that throws + // is `heal: 'none'` with the error in `detail` — flakiness can never + // buy a rebuild, and a thrown check never changes `serving` (the + // provider's serving verdict is composed by the provider, not inferred + // from a probe that failed to run). This catch used to synthesize + // `heal: 'rebuild'` — the read-triggered dark-rebuild lever one + // transient exception away — while the native composer said 'none'; + // two components disagreeing on what a throw means is how a flaky + // probe became an outage. `healthy: false` stays: an unrunnable probe + // is a named, loud, unverified state, never a clean bill. + const name = typeof (provider as { name?: string })?.name === 'string' + ? (provider as { name: string }).name + : 'unknown' reports.push({ - provider: 'unknown', + provider: name, healthy: false, - serving: false, + serving: true, invariants: [ { name: 'validate-invariants-threw', holds: false, detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`, - heal: 'rebuild' + heal: 'none' } ], checkedAt: Date.now(), diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index ce3236e9..78e310f3 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1200,10 +1200,23 @@ export interface RelateManyParams { */ export interface RepairFamilyReport { family: string + /** The family was actually examined (false = skipped; see `skipped`/`reason`). */ checked: boolean + /** Items re-posted / corrected in place — the incremental heal count. */ healed: number + /** + * What the check found missing or divergent, when it can name it: an exact + * count plus a capped sample of ids (never the whole list — a report is a + * verdict, not a dump). Absent when the family has nothing to name. + */ + missing?: { count: number; sample: string[] } + /** A full generational rebuild of this family ran (as opposed to an incremental heal). */ + rebuilt?: boolean detail?: string + /** Why the family was not checked (`checked: false`). */ skipped?: string + /** Why the outcome is what it is when neither `detail` nor `skipped` says it. */ + reason?: string } /** The full receipt returned by repairIndex(). */ diff --git a/tests/unit/validate-invariants-delegation.test.ts b/tests/unit/validate-invariants-delegation.test.ts index 45e12ccd..a5def81f 100644 --- a/tests/unit/validate-invariants-delegation.test.ts +++ b/tests/unit/validate-invariants-delegation.test.ts @@ -77,6 +77,31 @@ describe('validateIndexConsistency delegates to provider validateInvariants() (P delete brain.index.validateInvariants }) + it('ONE CONTRACT FOR A THROWING PROBE: heal is none (flakiness never buys a rebuild) and serving is not withheld', async () => { + // The probe that fails to RUN must never be read as "the index is broken, + // rebuild it" — that synthesized heal:'rebuild' was the dark-rebuild lever + // one transient exception away, and the native composer already said + // 'none' for the same event. Both engines now agree: named, loud, + // unverified — and never a rebuild, never a withheld serve. + brain.index.validateInvariants = async () => { throw new Error('transient: mmap window busy') } + const v = await brain.validateIndexConsistency() + const thrown = v.providers?.find((p: ProviderInvariantReport) => + p.invariants.some((i) => i.name === 'validate-invariants-threw') + ) + expect(thrown).toBeDefined() + expect(thrown!.healthy).toBe(false) + expect(thrown!.serving).toBe(true) + const inv = thrown!.invariants.find((i) => i.name === 'validate-invariants-threw')! + expect(inv.holds).toBe(false) + expect(inv.heal).toBe('none') + expect(inv.detail).toMatch(/transient: mmap window busy/) + // No provider report in the set recommends a rebuild for this event. + expect( + v.providers!.flatMap((p: ProviderInvariantReport) => p.invariants).some((i) => i.heal === 'rebuild') + ).toBe(false) + delete brain.index.validateInvariants + }) + it('providers without validateInvariants() are omitted (JS baseline unchanged)', async () => { const v = await brain.validateIndexConsistency() expect(v.providers).toBeUndefined() From 4176439ba3370206354c30139557e45fa9258653 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 24 Aug 2026 09:58:45 -0700 Subject: [PATCH 003/150] test(fold-checkpoint): the ARM-AT-FLIP pin arms its crash instead of racing the pending-flush timer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin wrote a post-flip row, abandoned the brain as crashed, and then deleted the row's canonical bytes to prove the first post-flip boot folds BOUNDED above the flip's stamp. Between the write's ack and the abandon sat the store's 50ms pending-flush timer: on a loaded box (the plant lane) the flush won, barrier-synced the row and advanced the checkpoint over it — and the fold, correctly bounded, did not restore bytes the test had destroyed after they were stamped durable. Green locally, red on the plant: the engine was right, the pin was timing-dependent. The crash is now armed at exactly singleop-after-fact-append: the fact is appended and at-ack synced, no flush is ever scheduled, the stamp provably still reads the flip's value when the pre-flip bytes are dropped, and the post-flip row's bytes — which lived only in the pending tier's RAM — are lost for real, not synthetically. The reopen must re-materialize it from its fact and must not restore the pre-flip row. --- .../integration/fold-checkpoint-bound.test.ts | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/integration/fold-checkpoint-bound.test.ts b/tests/integration/fold-checkpoint-bound.test.ts index 2f21248b..a02d75e0 100644 --- a/tests/integration/fold-checkpoint-bound.test.ts +++ b/tests/integration/fold-checkpoint-bound.test.ts @@ -23,6 +23,7 @@ import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { abandonAsCrashed, + armCrash, dropCanonicalNoun, makeTempDir, openBrain, @@ -177,15 +178,33 @@ describe('fold-checkpoint bound — crash recovery folds (checkpoint, head], nev expect(founded, 'checkpoint founded at flip').toBe(committedOf(brain)) // First post-flip boot, unclean (the production first-restart shape): - // a post-flip write above the checkpoint is restored FROM ITS AT-ACK FACT - // (deliberately NOT flushed — a flush would barrier-sync it and advance - // the stamp over it, making its loss synthetic); the pre-flip row (its - // baseline fact ≤ checkpoint, its bytes barrier-synced at the flip) is - // OUTSIDE the fold — vaporizing it synthetically proves the bound. - const postFlip = await brain.add({ data: 'post-flip write', type: NounType.Document, metadata: { era: 'log' } }) + // a post-flip write above the checkpoint is restored FROM ITS AT-ACK FACT; + // the pre-flip row (its baseline fact ≤ checkpoint, its bytes barrier- + // synced at the flip) is OUTSIDE the fold — vaporizing it synthetically + // proves the bound. + // + // THE CRASH IS ARMED, NOT RACED. The post-flip write "dies" at exactly + // `singleop-after-fact-append`: its fact is in the log and at-ack synced, + // and NO pending flush was ever scheduled — so the checkpoint provably + // still reads the flip's stamp when the bytes are dropped. The earlier + // shape (`add()` then abandon) raced the store's 50ms pending-flush + // timer: on a loaded box the flush won, barrier-synced the row, advanced + // the stamp over it — and the fold, CORRECTLY bounded, did not restore + // bytes the test had synthetically destroyed after they were stamped + // durable. The plant lane caught it; the engine was right, the pin was + // timing-dependent. + const postFlip = `post-flip-${Date.now().toString(36)}-0000-4000-8000-000000000000` + const arm = armCrash(brain, 'singleop-after-fact-append') + await expect( + brain.add({ id: postFlip, data: 'post-flip write', type: NounType.Document, metadata: { era: 'log' } }) + ).rejects.toThrow('simulated process crash at singleop-after-fact-append') + expect(arm.fired).toContain('singleop-after-fact-append') + expect(readCheckpoint(dir), 'the stamp did not move — nothing flushed after the flip').toBe(founded) await abandonAsCrashed(liveBrains.pop()!) + // The post-flip row's canonical bytes lived only in the pending tier's + // RAM (written at flush, never reached) — the crash takes them for real; + // nothing to drop. Only the pre-flip row is vaporized synthetically. dropCanonicalNoun(dir, preFlip) - dropCanonicalNoun(dir, postFlip) const reopened = await openBrain(dir, { logAuthority: 'adopt' }) liveBrains.push(reopened) From dcbad1765a2e3e6d100494b2074dc273917d949a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 24 Aug 2026 10:06:23 -0700 Subject: [PATCH 004/150] =?UTF-8?q?chore(release):=20--source-only=20?= =?UTF-8?q?=E2=80=94=20a=20home-only=20prerelease=20mode=20(The=20Source,?= =?UTF-8?q?=20never=20the=20storefront)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A prerelease the other engine devDeps from our own registry while the two are being proven together must never reach npmjs: --source-only tags, lets CI publish to The Source, creates the release page, and SKIPS the storefront publish, the cross-registry verification and the docs push — loudly, at each step. Refused for a non-prerelease version: a public floor always ships byte-identical on both registries. --- scripts/release.sh | 118 ++++++++++++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 45 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 03d60ac2..5d03e66b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -15,6 +15,12 @@ NC='\033[0m' # No Color RELEASE_TYPE="${1:-patch}" # patch, minor, or major SKIP_TESTS=false DRY_RUN=false +# --source-only: the HOME leg only — tag, CI's publish to The Source, and the +# release page; NO storefront (npmjs) publish, NO pair verification, NO docs +# push. The pair-gate shape: a prerelease the fleet's other engine devDeps +# from our own registry while the pair is proven, never a public artifact. +# Refused for a non-prerelease version — a public floor is always a pair. +SOURCE_ONLY=false for arg in "$@"; do case $arg in @@ -24,6 +30,9 @@ for arg in "$@"; do --dry-run) DRY_RUN=true ;; + --source-only) + SOURCE_ONLY=true + ;; esac done @@ -100,7 +109,7 @@ else ;; *) echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}" - echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run]" + echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run] [--source-only (prereleases only)]" exit 1 ;; esac @@ -119,6 +128,13 @@ echo -e "${BLUE}New version: ${NEW_VERSION}${NC}" if [ "$PRERELEASE" = true ]; then echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}" fi +if [ "$SOURCE_ONLY" = true ]; then + if [ "$PRERELEASE" != true ]; then + echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}" + exit 1 + fi + echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}" +fi echo "" if [ "$DRY_RUN" = true ]; then @@ -221,46 +237,50 @@ else exit 1 fi -echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}" -# BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download -# the tarball The Source serves and publish that file, never a fresh local pack -# (a local rebuild can differ byte-wise, and the fleet verifies the pair by -# shasum across registries). -STOREFRONT_TMP="$(mktemp -d)" -(cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null) -SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)" -echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}" -npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" -rm -rf "$STOREFRONT_TMP" -# Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. -npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true -# Verify the pair is byte-identical by registry-reported shasum — divergence -# here means the storefront leg must be treated as failed, loudly. RETRIED -# with raw curl: npmjs metadata propagates with a lag measured in minutes, -# and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a -# raw curl of the registry document already confirmed byte-identity. The -# probe now reads the registry JSON directly (no npm cache in the path) and -# gives propagation up to 5 minutes before calling the pair divergent. -NPMJS_VERIFY_ATTEMPTS=20 -NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace -SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable") -PAIR_IDENTICAL=false -for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do - NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \ - | node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \ - || echo "") - if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then - PAIR_IDENTICAL=true - break - fi - echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}" - sleep "$NPMJS_VERIFY_INTERVAL_S" -done -if [ "$PAIR_IDENTICAL" = true ]; then - echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n" +if [ "$SOURCE_ONLY" = true ]; then + echo -e "${YELLOW}9️⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n" else - echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n" - exit 1 + echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}" + # BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download + # the tarball The Source serves and publish that file, never a fresh local pack + # (a local rebuild can differ byte-wise, and the fleet verifies the pair by + # shasum across registries). + STOREFRONT_TMP="$(mktemp -d)" + (cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null) + SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)" + echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}" + npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" + rm -rf "$STOREFRONT_TMP" + # Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. + npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true + # Verify the pair is byte-identical by registry-reported shasum — divergence + # here means the storefront leg must be treated as failed, loudly. RETRIED + # with raw curl: npmjs metadata propagates with a lag measured in minutes, + # and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a + # raw curl of the registry document already confirmed byte-identity. The + # probe now reads the registry JSON directly (no npm cache in the path) and + # gives propagation up to 5 minutes before calling the pair divergent. + NPMJS_VERIFY_ATTEMPTS=20 + NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace + SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable") + PAIR_IDENTICAL=false + for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do + NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \ + | node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \ + || echo "") + if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then + PAIR_IDENTICAL=true + break + fi + echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}" + sleep "$NPMJS_VERIFY_INTERVAL_S" + done + if [ "$PAIR_IDENTICAL" = true ]; then + echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n" + else + echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n" + exit 1 + fi fi # Step 11: Release object on The Source (presentational — the tag, CHANGELOG, @@ -283,16 +303,24 @@ fi # DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — # that already happened) when a push errors, so the docs site never # silently trails npm. -echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" -if node scripts/push-docs.js; then - echo -e "${GREEN}✅ Docs push step done${NC}\n" +if [ "$SOURCE_ONLY" = true ]; then + echo -e "${YELLOW}1️⃣2️⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n" else - echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" + echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" + if node scripts/push-docs.js; then + echo -e "${GREEN}✅ Docs push step done${NC}\n" + else + echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" + fi fi echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" -echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}" +if [ "$SOURCE_ONLY" = true ]; then + echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}" +else + echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}" +fi echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" From a1376e4a2c9c6645d5aa384a8b9b15f6bd59fe93 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 24 Aug 2026 10:46:07 -0700 Subject: [PATCH 005/150] =?UTF-8?q?ci(publish):=20the=20home=20dist-tag=20?= =?UTF-8?q?follows=20the=20version=20=E2=80=94=20a=20prerelease=20publishe?= =?UTF-8?q?s=20under=20'rc'=20and=20never=20moves=20'latest'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .forgejo/workflows/publish-source.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml index 8220bac9..a08875ae 100644 --- a/.forgejo/workflows/publish-source.yml +++ b/.forgejo/workflows/publish-source.yml @@ -34,7 +34,16 @@ jobs: SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" VERSION="$(node -p "require('./package.json').version")" - echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry..." + # The dist-tag follows the version: a prerelease (any hyphen — + # 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' — + # every consumer resolving 'latest' from this registry would otherwise + # be handed a release candidate. Same rule scripts/release.sh applies + # to the storefront leg. + NPM_TAG="latest" + case "$VERSION" in + *-*) NPM_TAG="rc" ;; + esac + echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." TMPRC="$(mktemp)" chmod 600 "$TMPRC" @@ -47,7 +56,7 @@ jobs: # this tag's checkout already carries the version being published — # nothing here re-derives it from the tag name. PUBLISH_OK=true - if ! npm publish --tag latest --userconfig "$TMPRC"; then + if ! npm publish --tag "$NPM_TAG" --userconfig "$TMPRC"; then PUBLISH_OK=false fi From a8b5ca0c8fcc0e5b12229db52f147478cc28c124 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 24 Aug 2026 10:46:55 -0700 Subject: [PATCH 006/150] chore(release): 10.4.0-rc.1 --- CHANGELOG.md | 13 +++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f99584e5..cd64d88f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) + +- ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) +- chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) +- test(fold-checkpoint): the ARM-AT-FLIP pin arms its crash instead of racing the pending-flush timer (4176439b) +- fix(health): one contract for a throwing probe — heal is none, serving is not withheld; repair report gains missing/rebuilt/reason (116550eb) +- feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete (7c8c8be3) +- fix(delete): the null-metadata skip closes — index legs run id-keyed or narrate, never silently strand postings (607e9f54) +- feat(repair): repairIndex returns the per-family receipt and narrates its summary (8d45f964) +- fix(reads): the readiness gate guards every index read surface — serving empty from a not-ready provider is unrepresentable (40e7119b) +- ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) + + ### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18) - docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) diff --git a/package-lock.json b/package-lock.json index afce417d..8cee0038 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.3.1", + "version": "10.4.0-rc.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.3.1", + "version": "10.4.0-rc.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 75e5bfbc..ef803c11 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.3.1", + "version": "10.4.0-rc.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From f8f64780b11a305084348b9b0a92d4a33f0e43c1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 24 Aug 2026 12:45:51 -0700 Subject: [PATCH 007/150] =?UTF-8?q?feat(health):=20the=20gate=20reads=20th?= =?UTF-8?q?e=20named=20report=20=E2=80=94=20reads=20refuse=20loudly,=20nev?= =?UTF-8?q?er=20rebuild;=20open=20serves=20before=20it=20returns;=20the=20?= =?UTF-8?q?ceremony=20door?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read gate stops consulting the unnamed isReady() boolean: every provider may expose healthReport() (sync, O(1), composed from exact ledgers — HealthReport with a monotonic generation, per-invariant source ledger|deep|unledgered, missing {count, sample}), and one readiness authority (assessProviderHealth) derives the verdict. Unledgered families are UNKNOWN — never healthy, never broken; a report that throws is a loud not-ready, never a shrug. Reads at the four index choke points refuse with the typed NotReady errors, narrated once per (provider, generation) — a read NEVER starts a store walk: - the first-read lazy build retires (open builds instead, regardless of size — the ≥10k deferral and the "lazy loading on first query" branch go; disableAutoRebuild is re-meant honestly in its docs); - the verify*Live read-path rebuild triggers retire (refuse-or-serve); - the read-time consistency probe that could launch a dark rebuild from an ordinary find() retires; - repairIndex({ rebuild: ['metadata'|'graph'|'vector'] | 'all' }) is the one explicit door: rebuilds the named leg unconditionally and reports rebuilt per family; bare repairIndex() stays report-driven. test(lifecycle): the biography lane — a store's whole life, refereed tests/lifecycle/: an independent shadow model referees every read after every chapter (founding, a working day, clean restart, crash, repair, second life). Chapters 1-3 green. Chapters 4-6 assert the true contract and are marked .fails as a release-blocking finding (the kill-matrix convention): after a crash + adopt reopen the metadata index computes its 'catchup' watermark verdict and nothing consumes it — find() serves the pre-crash index while canonical and counts recover. The catchup wiring is the cure; a passing .fails will force the marker's removal. The lane runs in the integration gate (config + coverage guard). --- docs/api/README.md | 23 + src/brainy.ts | 792 ++++++++---------- src/index.ts | 4 + src/plugin.ts | 102 +++ src/types/brainy.types.ts | 14 +- src/utils/indexReadiness.ts | 117 +++ tests/configs/vitest.integration.config.ts | 3 + .../cold-graph-connected-8.0.test.ts | 129 ++- tests/integration/health-gate.test.ts | 352 ++++++++ tests/lifecycle/README.md | 16 + tests/lifecycle/biography.test.ts | 429 ++++++++++ tests/lifecycle/biographyHarness.ts | 389 +++++++++ tests/unit/brainy/lazy-notready-honor.test.ts | 94 ++- .../brainy/metadata-provider-contract.test.ts | 59 +- tests/unit/brainy/migration-deference.test.ts | 41 +- tests/unit/metadata-cold-read-guard.test.ts | 40 +- tests/unit/test-suite-coverage-guard.test.ts | 3 + tests/unit/utils/indexReadiness.test.ts | 153 ++++ tests/unit/vector-cold-read-guard.test.ts | 52 +- 19 files changed, 2160 insertions(+), 652 deletions(-) create mode 100644 tests/integration/health-gate.test.ts create mode 100644 tests/lifecycle/README.md create mode 100644 tests/lifecycle/biography.test.ts create mode 100644 tests/lifecycle/biographyHarness.ts create mode 100644 tests/unit/utils/indexReadiness.test.ts diff --git a/docs/api/README.md b/docs/api/README.md index 4ca84364..fb9cc920 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -1831,6 +1831,29 @@ const semanticOnly = await brain.getStats({ excludeVFS: true }) --- +### `repairIndex(options?)` → `Promise` + +The ceremony door for index repair. Bare `repairIndex()` is report-driven: it +prunes orphaned containers, recomputes count rollups, reconciles VFS +containment, and rebuilds only a derived-index family whose own health check +asks for it. Pass `options.rebuild` to force one or more families to rebuild +UNCONDITIONALLY — no health check is consulted — when an operator has +independent reason to reconcile a family regardless of what it self-reports. + +```typescript +// Report-driven: only heals what actually needs it +const report = await brain.repairIndex() +console.log(report.healedTotal, report.families) + +// Explicit: force the graph adjacency to rebuild from canonical, unconditionally +await brain.repairIndex({ rebuild: ['graph'] }) + +// Explicit: force all three derived indexes to rebuild +await brain.repairIndex({ rebuild: 'all' }) +``` + +--- + ## Lifecycle ### Initialization diff --git a/src/brainy.ts b/src/brainy.ts index db332e18..c8912e26 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -197,7 +197,7 @@ import { import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' -import { assessIndexReadiness } from './utils/indexReadiness.js' +import { assessIndexReadiness, assessProviderHealth } from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { @@ -634,8 +634,6 @@ export class Brainy implements BrainyInterface { /** One-shot guard so the degraded-reads warning fires once per degraded window * (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */ private _degradedReadWarned = false - /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */ - private _metadataConsistencyProbed = false /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */ private _graphAdjacencyVerified = false /** Re-entrancy guard: a verify (rebuild → reads) is in flight. */ @@ -804,11 +802,20 @@ export class Brainy implements BrainyInterface { // applies only to instances that were never closed. private closed = false - // Lazy rebuild state (Production-scale lazy loading) - // Prevents race conditions when multiple queries trigger rebuild simultaneously - private lazyRebuildInProgress = false + // Index-build-at-open state. `lazyRebuildCompleted` predates the health-gate + // law (it named a first-QUERY lazy rebuild) and stays for `getIndexStatus()` + // API compatibility, but its truth changed: a needed rebuild now runs + // unconditionally at open() (see `rebuildIndexesIfNeeded`), never deferred to + // a read, so this simply flips true once that open-time step has run. + // `lazyRebuildInProgress` / `lazyRebuildPromise` (the first-query rebuild's + // concurrency guard) are retired with the lazy-build path they served — + // `ensureIndexesLoaded()` is a read-time CHECK now, never a build. private lazyRebuildCompleted = false - private lazyRebuildPromise: Promise | null = null + + // Read-gate narration dedup: a degraded-but-serving or not-ready health + // report narrates via prodLog.warn ONCE per (provider, report.generation) — + // never once per read. Keyed on the provider instance itself. + private _lastNarratedHealthGeneration = new Map() constructor(config?: BrainyConfig) { // The reserved-field write policy died with the field-addressing law: @@ -1444,8 +1451,12 @@ export class Brainy implements BrainyInterface { }).backfillBlobHistoryRefCountsIfNeeded() } - // Rebuild indexes if needed for existing data + // Rebuild indexes if needed for existing data. Runs to completion before + // init() returns — there is no more first-query lazy path, so the flag + // below (kept for getIndexStatus() API compatibility) simply flips true + // once this open-time step has run. await this.rebuildIndexesIfNeeded() + this.lazyRebuildCompleted = true // Check for pending data migrations await this.checkMigrations() @@ -3976,8 +3987,9 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. Fast path after the latch is one boolean. - await this.ensureIndexesLoaded() + // provider for 15 minutes. A CHECK only — it never builds; throws a typed + // NotReady error if a provider's health report says it isn't serving. + this.ensureIndexesLoaded() const entityInt = this.graphEntityInt(uuid) if (entityInt === undefined) return [] const neighborInts = await this.graphIndex.getNeighbors(entityInt, options) @@ -4005,72 +4017,59 @@ export class Brainy implements BrainyInterface { /** * @description Verify that the graph adjacency is actually LIVE before a graph read trusts * its result. A native graph index can load its relationship COUNT (manifest) on a cold open - * of a LARGE brain (≥10k nouns, which skips the eager index rebuild) but NOT its - * source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even though - * edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would serve - * that `[]` as if it were truth. + * but NOT its source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even + * though edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would + * serve that `[]` as if it were truth. * - * Two detection strategies, in order of honesty: - * - **Preferred (8.0 contract):** the provider exposes a sync `isReady()` that is true ONLY - * when the edges are loaded. `false` → hydrate the id-mapper (a native int adjacency - * resolves endpoints through it), rebuild from storage, and re-check `isReady()`; if it is - * still `false`, throw {@link GraphIndexNotReadyError} rather than returning `[]`. - * - **Fallback (providers without `isReady()`):** a GLOBAL known-edge sample (a real + * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the dark-rebuild + * failure mode this contract retires (open() alone owns building; see + * {@link rebuildIndexesIfNeeded}). Two detection strategies, in order of honesty: + * - **Preferred:** {@link assessProviderHealth} — the provider's named `healthReport()` when + * exposed, else its sync `isReady()`. Not serving → THROW {@link GraphIndexNotReadyError} + * naming the reasons, immediately — no rebuild attempt. + * - **Fallback (providers with neither signal):** a READ-ONLY GLOBAL known-edge sample (a real * persisted verb's `sourceId`, which by definition HAS an outgoing edge) — NOT any queried * anchor, because brainy cannot cheaply tell "adjacency unloaded" from "this node is - * genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, the - * adjacency did not load: rebuild and re-probe; if even that fails, throw. + * genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, THROW — + * the probe refuses loudly; it does not self-heal. * - * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing - * to verify), or `'rebuilt'` when a cold-unloaded adjacency was just healed from storage — - * in which case callers that observed an empty result must RE-RUN their collection. - * @throws {GraphIndexNotReadyError} when the index claims edges but cannot serve a known - * persisted edge (or stays not-ready) even after a rebuild. + * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing to + * verify). + * @throws {GraphIndexNotReadyError} when the index is not serving, or claims edges but cannot + * serve a known persisted edge. */ - private async verifyGraphAdjacencyLive(): Promise<'live' | 'rebuilt'> { + private async verifyGraphAdjacencyLive(): Promise<'live'> { if (this._graphAdjacencyVerified) return 'live' // Coordinated migration LOCK (#18): while the graph provider owns a locked - // rebuild-from-canonical, brainy must NOT fire its own graphIndex.rebuild() - // on a read — that would race the provider's in-place rebuild. The data-plane - // lock (awaitMigrationLock in ensureInitialized) already makes callers wait, - // so this is normally unreachable mid-migration; the guard is defensive. It + // rebuild-from-canonical, brainy must NOT judge it here — the provider owns + // its index until it verifies-and-swaps. The data-plane lock + // (awaitMigrationLock in ensureInitialized) already makes callers wait, so + // this is normally unreachable mid-migration; the guard is defensive. It // deliberately does NOT set `_graphAdjacencyVerified`, so the real verify runs // once the migration clears. if (this.providerIsMigrating(this.graphIndex)) return 'live' - // Re-entrancy: rebuild() can trigger reads (neighbors/related) that call back into this - // guard. While a verify is in flight, short-circuit so we cannot recurse into rebuild(). + // Re-entrancy: a fallback probe below calls getNeighbors(), which does not + // re-enter this guard, but the short-circuit is kept defensively cheap. if (this._graphAdjacencyVerifying) return 'live' this._graphAdjacencyVerifying = true try { - const gi = this.graphIndex as GraphAdjacencyIndex & { isReady?: () => boolean } - - // ── Strategy 1: honest isReady() signal (cortex >= 2.7.8 / 3.0) ────────── - if (typeof gi.isReady === 'function') { - if (gi.isReady()) { + // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── + const assessment = assessProviderHealth(this.graphIndex) + if (assessment.via === 'health-report' || assessment.via === 'is-ready') { + if (assessment.readiness === 'ready') { this._graphAdjacencyVerified = true return 'live' } - // Not ready: the edges did not load on open. Hydrate the id-mapper, then rebuild. - if (!this.config.silent) { - console.warn( - `[Brainy] Graph adjacency reports not-ready (isReady() === false) — the persisted ` + - `adjacency did not load on open. Rebuilding from storage…` - ) - } - await this.hydrateIdMapperForGraphRebuild() - await this.graphIndex.rebuild() - if (gi.isReady()) { - this._graphAdjacencyVerified = true - return 'rebuilt' - } throw new GraphIndexNotReadyError( - `Graph adjacency index reports not-ready even after a rebuild — the persisted ` + - `adjacency could not be loaded. find({ connected }), neighbors() and related() ` + - `cannot be served reliably for this brain.` + `Graph adjacency index is not serving (via ${assessment.via}): ` + + `${assessment.reasons.join('; ') || 'not ready'}. find({ connected }), neighbors() and ` + + `related() refuse rather than serve an empty result — rebuild via ` + + `repairIndex({ rebuild: ['graph'] }) or reopen the brain.` ) } - // ── Strategy 2: known-edge-sample probe (providers without isReady()) ──── + // ── Strategy 2: known-edge-sample probe (providers with neither signal) ─ + // READ-ONLY — refuses loudly on failure; never calls rebuild(). const claimed = await this.graphIndex.size() if (!claimed || claimed <= 0) return 'live' // no edges claimed — nothing to verify @@ -4086,10 +4085,9 @@ export class Brainy implements BrainyInterface { // the sample is not one of this brain's own edges — e.g. a shared on-disk store reused // across instances surfaces a foreign verb whose UUID this brain's resident mapper never // interned. We cannot prove a cold-unloaded adjacency from such a sample, so treat it as - // INCONCLUSIVE: mark verified and return 'live' rather than rebuilding/throwing. (The honest - // cold-load signal for native providers is isReady(), checked above; the JS baseline keeps - // its mapper resident, so its OWN edges always resolve — the targeted 7.x failure mode, - // "mapper loaded but adjacency empty", still resolves the source and is detected below.) + // INCONCLUSIVE: mark verified and return 'live' rather than throwing. (The honest cold-load + // signal for native providers is Strategy 1, checked above; the JS baseline keeps its mapper + // resident, so its OWN edges always resolve.) const sourceInt = this.graphEntityInt(verb.sourceId) if (sourceInt === undefined) { this._graphAdjacencyVerified = true @@ -4097,37 +4095,23 @@ export class Brainy implements BrainyInterface { } // Ask the adjacency for ONE neighbor of the (mapped) known-edge source. - const probeKnownSource = async (): Promise => - (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0 - - if (await probeKnownSource()) { + const hasNeighbor = (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0 + if (hasNeighbor) { this._graphAdjacencyVerified = true return 'live' // adjacency is live — the common case } // INCONSISTENT: the index reports edges but a KNOWN-mapped persisted edge's source has none → - // the adjacency did not load on open. Hydrate the mapper and rebuild from storage. - if (!this.config.silent) { - console.warn( - `[Brainy] Graph adjacency reports ${claimed} relationship(s) but a persisted edge ` + - `resolves to none — the persisted adjacency did not load on open. Rebuilding from storage…` - ) - } - await this.hydrateIdMapperForGraphRebuild() - await this.graphIndex.rebuild() - - if (await probeKnownSource()) { - this._graphAdjacencyVerified = true - return 'rebuilt' - } + // the adjacency did not load. Refuse loudly — never rebuild from a read. throw new GraphIndexNotReadyError( - `Graph adjacency index reports ${claimed} relationship(s) but returns no edges even ` + - `after a rebuild — the persisted adjacency could not be loaded. find({ connected }), ` + - `neighbors() and related() cannot be served reliably for this brain.` + `Graph adjacency index reports ${claimed} relationship(s) but a persisted edge's source ` + + `resolves to none — the persisted adjacency did not load. find({ connected }), ` + + `neighbors() and related() refuse rather than serve an empty result — rebuild via ` + + `repairIndex({ rebuild: ['graph'] }) or reopen the brain.` ) } catch (err) { if (err instanceof GraphIndexNotReadyError) throw err - // A transient probe/rebuild failure must not break the actual query NOR be + // A transient probe failure must not break the actual query NOR be // masked as "no data". Allow a re-check on the next graph read and fall through. this._graphAdjacencyVerified = false if (!this.config.silent) { @@ -4144,27 +4128,51 @@ export class Brainy implements BrainyInterface { * On a cold open a native metadata provider can report data yet not serve its * `where` postings, so `find({ where })` silently returns `[]` — the exact * failure a downstream deployment reported (cold reads blanking filtered pages - * after every restart). This one-shot guard, run on the first FILTERED `find()`, - * closes that: it takes a KNOWN persisted entity + one of its plain field values - * and asks the index to resolve it. If the index returns the known id the field - * postings are live (the common case, and the ONLY cost on a warm brain — one - * O(1) probe). If it does not, the postings did not load: brainy rebuilds the - * index from the canonical records and re-probes; if it STILL cannot serve the - * known value it throws a loud {@link MetadataIndexNotReadyError} rather than - * let a silent `[]` stand. Inconclusive cases (empty store, no plain field to - * probe, a shared store surfacing a foreign entity) are treated as live — never - * a false rebuild. A migrating provider is skipped (it owns its locked rebuild). - * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. + * after every restart). + * + * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the + * dark-rebuild failure mode this contract retires (open() alone owns + * building; see {@link rebuildIndexesIfNeeded}). Two detection strategies: + * - **Preferred:** {@link assessProviderHealth} — the provider's named + * `healthReport()` when exposed, else its sync `isReady()`. Not serving → + * THROW {@link MetadataIndexNotReadyError} naming the reasons, immediately. + * - **Fallback (providers with neither signal):** a READ-ONLY known-value + * probe, run on the first FILTERED `find()`: take a KNOWN persisted entity + * + one of its plain field values and ask the index to resolve it. If the + * index does not return the known id, THROW — the probe refuses loudly; + * it does not self-heal. Inconclusive cases (empty store, no plain field + * to probe, a shared store surfacing a foreign entity) are treated as + * live — never a false throw. A migrating provider is skipped (it owns + * its locked rebuild). + * @returns `'live'` when the index serves. */ - private async verifyMetadataLive(): Promise<'live' | 'rebuilt'> { + private async verifyMetadataLive(): Promise<'live'> { if (this._metadataVerified) return 'live' // Migration LOCK (#18): a migrating provider owns its in-place rebuild — do // not race it. Defensive; the data-plane lock already gates callers upstream. if (this.providerIsMigrating(this.metadataIndex)) return 'live' - // Re-entrancy: rebuild() can trigger reads that call back into this guard. + // Re-entrancy: the fallback probe below calls filterIdsBelted(), which + // re-enters ensureIndexesLoaded() (a cheap CHECK) but not this guard. if (this._metadataVerifying) return 'live' this._metadataVerifying = true try { + // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── + const assessment = assessProviderHealth(this.metadataIndex) + if (assessment.via === 'health-report' || assessment.via === 'is-ready') { + if (assessment.readiness === 'ready') { + this._metadataVerified = true + return 'live' + } + throw new MetadataIndexNotReadyError( + `Metadata field index is not serving (via ${assessment.via}): ` + + `${assessment.reasons.join('; ') || 'not ready'}. find({ where }) and other filtered ` + + `reads refuse rather than serve an empty result — rebuild via ` + + `repairIndex({ rebuild: ['metadata'] }) or reopen the brain.` + ) + } + + // ── Strategy 2: known-value probe (providers with neither signal) ────── + // READ-ONLY — refuses loudly on failure; never calls rebuild(). // A KNOWN persisted entity + one plain field to probe. Sample a few so a // system-only entity (e.g. the VFS root) doesn't make every open inconclusive. const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) @@ -4186,7 +4194,7 @@ export class Brainy implements BrainyInterface { return ids.includes(p.id) } catch { // FIELD_NOT_INDEXED for a field a persisted entity actually holds is - // itself the cold/broken signal — treat as not-serving (→ rebuild). + // itself the cold/broken signal — treat as not-serving. return false } } @@ -4196,26 +4204,15 @@ export class Brainy implements BrainyInterface { return 'live' // field postings are live — the common case } - if (!this.config.silent) { - console.warn( - `[Brainy] Metadata field index returns no match for a known persisted value of ` + - `'${p.field}' — the field postings did not load on open. Rebuilding from storage…` - ) - } - await this.metadataIndex.rebuild() - - if (await probeServes()) { - this._metadataVerified = true - return 'rebuilt' - } throw new MetadataIndexNotReadyError( - `Metadata field index cannot serve a known persisted value of '${p.field}' even after ` + - `a rebuild — find({ where }) and other filtered reads cannot be served reliably for ` + - `this brain (a silent empty result would misrepresent existing data).` + `Metadata field index cannot serve a known persisted value of '${p.field}' — the field ` + + `postings did not load. find({ where }) and other filtered reads refuse rather than ` + + `serve an empty result — rebuild via repairIndex({ rebuild: ['metadata'] }) or reopen ` + + `the brain.` ) } catch (err) { if (err instanceof MetadataIndexNotReadyError) throw err - // A transient probe/rebuild failure must not break the query NOR mask as + // A transient probe failure must not break the query NOR mask as // "no data". Allow a re-check on the next filtered read and fall through. this._metadataVerified = false if (!this.config.silent) { @@ -4261,57 +4258,52 @@ export class Brainy implements BrainyInterface { * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded * its serving structure (the mmap/DiskANN graph) — so a pure semantic * `find({ query })` silently returns `[]`. A pure semantic query has - * `hasFilterCriteria === false`, so the metadata guard never fires; this guard - * closes that gap. Run one-shot on the first vector/proximity search: - * - **Preferred (honest signal):** the provider exposes `isReady()`. `false` - * → rebuild from storage, re-check; if still `false`, throw - * {@link VectorIndexNotReadyError} rather than serving `[]`. - * - **Fallback (no `isReady()`):** a KNOWN persisted vector (sampled + - * hydrated) is searched against the index; if it does not self-match, the - * serving structure did not load — rebuild + re-probe, else throw. + * `hasFilterCriteria === false`, so the metadata guard never fires; this + * guard closes that gap. Run one-shot on the first vector/proximity search. + * + * NEVER REBUILDS, NEVER WALKS THE STORE — a read-path rebuild is exactly the + * dark-rebuild failure mode this contract retires (open() alone owns + * building; see {@link rebuildIndexesIfNeeded}). Two detection strategies: + * - **Preferred:** {@link assessProviderHealth} — the provider's named + * `healthReport()` when exposed, else its sync `isReady()`. Not serving → + * THROW {@link VectorIndexNotReadyError} naming the reasons, immediately. + * - **Fallback (providers with neither signal):** a READ-ONLY KNOWN + * persisted vector (sampled + hydrated) is searched against the index; if + * it does not self-match, THROW — the probe refuses loudly; it does not + * self-heal. * Inconclusive cases (empty store, no probeable vector, `size()===0` — where - * the JS baseline's cold load is `ensureIndexesLoaded`'s job) are treated as - * live: never a false rebuild. A migrating provider is skipped (it owns its - * locked rebuild). - * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. + * the JS baseline is built at open) are treated as live: never a false + * throw. A migrating provider is skipped (it owns its locked rebuild). + * @returns `'live'` when the index serves. */ - private async verifyVectorLive(): Promise<'live' | 'rebuilt'> { + private async verifyVectorLive(): Promise<'live'> { if (this._vectorVerified) return 'live' // Migration LOCK (#18): a migrating provider owns its in-place rebuild. if (this.providerIsMigrating(this.index)) return 'live' - // Re-entrancy: rebuild() can trigger reads that call back into this guard. + // Re-entrancy: the fallback probe below calls index.search(), which does + // not re-enter this guard, but the short-circuit is kept defensively cheap. if (this._vectorVerifying) return 'live' this._vectorVerifying = true try { - // ── Strategy 1: honest isReady() signal (native provider) ────────────── - const readiness = assessIndexReadiness(this.index) - if (readiness !== 'unknown') { - if (readiness === 'ready') { + // ── Strategy 1: the health-report/isReady() authority — never rebuilds ── + const assessment = assessProviderHealth(this.index) + if (assessment.via === 'health-report' || assessment.via === 'is-ready') { + if (assessment.readiness === 'ready') { this._vectorVerified = true return 'live' } - // Not ready: the serving structure did not load on open. Rebuild. - if (!this.config.silent) { - console.warn( - `[Brainy] Vector index reports not-ready (isReady() === false) — the persisted ` + - `vector index did not load on open. Rebuilding from storage…` - ) - } - await this.index.rebuild() - if (assessIndexReadiness(this.index) === 'ready') { - this._vectorVerified = true - return 'rebuilt' - } throw new VectorIndexNotReadyError( - `Vector index reports not-ready even after a rebuild — semantic find({ query }) and ` + - `proximity search cannot be served reliably for this brain (a silent empty result ` + - `would misrepresent existing data).` + `Vector index is not serving (via ${assessment.via}): ` + + `${assessment.reasons.join('; ') || 'not ready'}. Semantic find({ query }) and ` + + `proximity search refuse rather than serve an empty result — rebuild via ` + + `repairIndex({ rebuild: ['vector'] }) or reopen the brain.` ) } - // ── Strategy 2: known-vector probe (providers without isReady()) ─────── + // ── Strategy 2: known-vector probe (providers with neither signal) ───── + // READ-ONLY — refuses loudly on failure; never calls rebuild(). const claimed = this.index.size() - if (!claimed || claimed <= 0) return 'live' // JS cold path is ensureIndexesLoaded's job + if (!claimed || claimed <= 0) return 'live' // JS cold path is built at open const probe = await this.pickVectorProbe() if (!probe) { @@ -4321,44 +4313,30 @@ export class Brainy implements BrainyInterface { } const p = probe - const probeServes = async (): Promise => { - // The failure mode we guard is the SILENT EMPTY result: a cold index that - // loaded its COUNT but not its serving structure returns `[]` for a - // known-present vector, while a warm index returns at least one hit. We - // check for a NON-EMPTY result, NOT an exact self-match — HNSW is - // approximate and `get()` may return a re-hydrated/normalized vector, so - // demanding the exact self as top-1 would false-positive on a perfectly - // healthy index (and wrongly rebuild → throw). - const hits = await this.index.search(p.vector, 1) - return hits.length > 0 - } + // The failure mode we guard is the SILENT EMPTY result: a cold index that + // loaded its COUNT but not its serving structure returns `[]` for a + // known-present vector, while a warm index returns at least one hit. We + // check for a NON-EMPTY result, NOT an exact self-match — HNSW is + // approximate and `get()` may return a re-hydrated/normalized vector, so + // demanding the exact self as top-1 would false-positive on a perfectly + // healthy index (and wrongly throw). + const hits = await this.index.search(p.vector, 1) void p.id // probe keyed on the vector; id retained for diagnostics only - if (await probeServes()) { + if (hits.length > 0) { this._vectorVerified = true return 'live' // serving structure is live — the common case } - if (!this.config.silent) { - console.warn( - `[Brainy] Vector index reports ${claimed} vector(s) but a known persisted vector ` + - `returns no results — the serving structure did not load on open. Rebuilding…` - ) - } - await this.index.rebuild() - - if (await probeServes()) { - this._vectorVerified = true - return 'rebuilt' - } throw new VectorIndexNotReadyError( `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` + - `results even after a rebuild — semantic find({ query }) cannot be served reliably ` + - `for this brain (a silent empty result would misrepresent existing data).` + `results — the serving structure did not load. Semantic find({ query }) refuses rather ` + + `than serve an empty result — rebuild via repairIndex({ rebuild: ['vector'] }) or ` + + `reopen the brain.` ) } catch (err) { if (err instanceof VectorIndexNotReadyError) throw err - // A transient probe/rebuild failure must not break the query NOR mask as + // A transient probe failure must not break the query NOR mask as // "no data". Allow a re-check on the next vector read and fall through. this._vectorVerified = false if (!this.config.silent) { @@ -6559,14 +6537,10 @@ export class Brainy implements BrainyInterface { // loader and cold-read probes below already defer to a migrating provider. await this.ensureInitialized({ needs: [] }) - // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) - // This is a production-safe, concurrency-controlled lazy load - await this.ensureIndexesLoaded() - - // One-shot cold-open self-heal: an O(1) probe of the metadata index (when the - // provider offers one) repairs an already-poisoned index on first read — the - // metadata counterpart of the graph cold-load guard. No-op for the JS index. - await this.ensureMetadataConsistencyProbed() + // READ-SURFACE READINESS GATE (see filterIdsBelted): a CHECK only — it + // never builds. open() already brought every provider to serving before + // init() returned; this throws a typed NotReady error if one isn't. + this.ensureIndexesLoaded() // Loudly flag a degraded derived index (failed init rebuild, or an // adopt-forward degraded commit) so a partial result is never mistaken for @@ -11851,10 +11825,10 @@ export class Brainy implements BrainyInterface { } /** - * Get index loading status (Diagnostic for lazy loading) + * Get index loading status (diagnostic) * - * Returns detailed information about index population and lazy loading state. - * Useful for debugging empty query results or performance troubleshooting. + * Returns detailed information about index population state. Useful for + * debugging empty query results or performance troubleshooting. * * @example * ```typescript @@ -11863,7 +11837,7 @@ export class Brainy implements BrainyInterface { * console.log(`Metadata Index: ${status.metadataIndex.entries} entries`) * console.log(`Graph Index: ${status.graphIndex.relationships} relationships`) * console.log(`Pending embeds: ${status.projections.semantic.pendingEmbeds}`) - * console.log(`Lazy rebuild completed: ${status.lazyRebuildCompleted}`) + * console.log(`Index build completed at open: ${status.lazyRebuildCompleted}`) * ``` */ @@ -11882,8 +11856,9 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. Fast path after the latch is one boolean. - await this.ensureIndexesLoaded() + // provider for 15 minutes. A CHECK only — it never builds; throws a typed + // NotReady error if a provider's health report says it isn't serving. + this.ensureIndexesLoaded() try { return await this.metadataIndex.getIdsForFilter(filter, opts) } catch (err) { @@ -11895,6 +11870,10 @@ export class Brainy implements BrainyInterface { async getIndexStatus(): Promise<{ initialized: boolean + /** `true` once open()'s index-build-if-needed step has run. Named for API + * compatibility with the retired first-query lazy-build path; a needed + * rebuild now always runs at open, never deferred to a read, so this is + * simply `initialized`'s index-build counterpart. */ lazyRebuildCompleted: boolean /** Deferred embeds not yet landed (MT5) — the eventual-vector-index backlog. */ pendingEmbeds: number @@ -14373,8 +14352,9 @@ export class Brainy implements BrainyInterface { // index read funnels through this helper, so the gate here makes // serve-while-not-ready UNREPRESENTABLE — a production store once acked // writes while every non-find() read served empty from a not-ready - // provider for 15 minutes. Fast path after the latch is one boolean. - await this.ensureIndexesLoaded() + // provider for 15 minutes. A CHECK only — it never builds; throws a typed + // NotReady error if a provider's health report says it isn't serving. + this.ensureIndexesLoaded() // 8.0 BigInt boundary: unmapped node → no relations. const nodeInt = this.graphEntityInt(nodeId) if (nodeInt === undefined) return [] @@ -14983,16 +14963,13 @@ export class Brainy implements BrainyInterface { // Cold-load guard: an empty connected set is suspicious. The native adjacency can report // size()>0 (or isReady()===false) on a cold open yet have loaded NO source→target edges — so - // traversal silently returns []. Re-verify against the honest isReady() signal (or, for older - // providers, a GLOBAL known-edge sample — NOT the queried anchor, which may be genuinely - // edgeless). If the adjacency was dead and a rebuild healed it, re-collect; if it stays dead, - // verifyGraphAdjacencyLive() throws GraphIndexNotReadyError. A genuinely edgeless anchor - // verifies 'live' and the empty result stands — no spurious rebuild/throw. + // traversal would silently return [] as if it were truth. Re-verify against the health-report/ + // isReady() authority (or, for older providers, a READ-ONLY GLOBAL known-edge sample — NOT the + // queried anchor, which may be genuinely edgeless): a dead adjacency throws + // GraphIndexNotReadyError here rather than serving the empty set as fact — verifyGraphAdjacencyLive + // never rebuilds, so a genuinely edgeless anchor simply verifies 'live' and the empty result stands. if (connectedIds.size === 0) { - const verdict = await this.verifyGraphAdjacencyLive() - if (verdict === 'rebuilt') { - await populate() - } + await this.verifyGraphAdjacencyLive() } // Filter existing results to only connected entities @@ -16228,111 +16205,59 @@ export class Brainy implements BrainyInterface { } /** - * Ensure indexes are loaded (Production-scale lazy loading) + * @description THE READ GATE. Every read choke point (getNeighborUuids, + * find, filterIdsBelted, getTypedNeighbors) calls this before touching a + * derived index. It is a CHECK, never a build: it asks each of the three + * providers (vector, metadata, graph) for its named health verdict via + * {@link assessProviderHealth} — the provider's own sync, O(1) + * `healthReport()` when exposed, else the `isReady()` / size-heuristic + * fallback — and either lets the read proceed or throws the matching typed + * `*NotReadyError` naming the provider and its failing reasons. It NEVER + * triggers a rebuild and NEVER walks the store: a needed rebuild is + * entirely open()'s job (see {@link rebuildIndexesIfNeeded}), which runs to + * completion before `init()` returns — so by the time any read reaches + * this gate, a healthy provider is already built. A migrating provider is + * deferred to exactly as before (it owns its own in-place rebuild). * - * Called by query methods (find, search, get, etc.) when disableAutoRebuild is true. - * Handles concurrent queries safely - multiple calls wait for same rebuild. - * - * Performance: - * - First query: Triggers rebuild (~50-200ms for 1K-10K entities) - * - Concurrent queries: Wait for same rebuild (no duplicate work) - * - Subsequent queries: Instant (0ms check, indexes already loaded) - * - * Production scale: - * - 1K entities: ~50ms - * - 10K entities: ~200ms - * - 100K entities: ~2s (streaming pagination) - * - 1M+ entities: Uses chunked lazy loading (per-type on demand) + * A report with something worth telling an operator (a failing invariant, + * whether serving or not, or a named `unledgered` family) narrates via + * `prodLog.warn` ONCE per (provider, `report.generation`) — never once per + * read — before any throw decision is made. */ - private async ensureIndexesLoaded(): Promise { - // Fast path: If rebuild already completed, return immediately (0ms) - if (this.lazyRebuildCompleted) { - return + private ensureIndexesLoaded(): void { + const providers: ReadonlyArray BrainyError]> = [ + ['vector', this.index, VectorIndexNotReadyError], + ['metadata', this.metadataIndex, MetadataIndexNotReadyError], + ['graph', this.graphIndex, GraphIndexNotReadyError] + ] + + for (const [name, provider, ErrorClass] of providers) { + // Migration LOCK (#18) deference: a migrating provider owns its own + // in-place rebuild — brainy must not judge (or race) it here. + if (this.providerIsMigrating(provider)) continue + + const assessment = assessProviderHealth(provider) + + if (assessment.reasons.length > 0 && assessment.report != null) { + const generation = assessment.report.generation + if (this._lastNarratedHealthGeneration.get(provider) !== generation) { + this._lastNarratedHealthGeneration.set(provider, generation) + prodLog.warn( + `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + + assessment.reasons.join('; ') + ) + } + } + + if (assessment.readiness === 'not-ready') { + throw new ErrorClass( + `${name} index is not serving (via ${assessment.via}): ` + + `${assessment.reasons.join('; ') || 'not ready'}. Reads refuse rather than serve an ` + + `empty result — open() builds the derived indexes; a read never does. Rebuild via ` + + `repairIndex({ rebuild: ['${name}'] }) or reopen the brain.` + ) + } } - - // If indexes already populated AND honestly serving, mark complete and skip. - // Honest gate: when a provider exposes isReady(), that REPLACES the size()>0 - // proxy (a native index can report a non-zero size while its serving structure - // is not loaded — the silent-empty cold-load class). A not-ready provider falls - // through so the rebuild path can load it; verifyVectorLive() is the query-time - // backstop either way. Providers without isReady() keep the size() heuristic - // (the JS index's size()>0 genuinely means loaded). - // - // ALL THREE providers vote (fleet-adoption find, SELF-ENGINE-PAIR-STANDARD): - // this gate used to assess ONLY the vector index, so a not-ready native - // METADATA provider (its strand report) never blocked the completion latch - // — under disableAutoRebuild the promised lazy first-query rebuild never - // fired and every find() silently returned [] on a populated store. A - // not-ready report from ANY provider now falls through to the rebuild. - const vectorReadiness = assessIndexReadiness(this.index) - const metadataReadiness = assessIndexReadiness(this.metadataIndex) - const graphReadiness = assessIndexReadiness(this.graphIndex) - const anyProviderNotReady = - vectorReadiness === 'not-ready' || - metadataReadiness === 'not-ready' || - graphReadiness === 'not-ready' - if ( - !anyProviderNotReady && - (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) - ) { - this.lazyRebuildCompleted = true - return - } - - // Migration LOCK (#18) deference: while the vector provider runs its one-time - // 7.x → 8.0 rebuild-from-canonical, a first query must NOT trigger brainy's - // force-rebuild — the provider owns that index. Normally unreachable here: the - // data-plane lock (awaitMigrationLock) makes the caller wait upstream, so a - // query only reaches this point once the migration has cleared. Defensive - // (no `lazyRebuildCompleted` latch) so the check re-runs: once the provider - // clears the lock, `index.size() > 0` above ends the lazy path normally. - if (this.providerIsMigrating(this.index)) { - return - } - - // Concurrency control: If rebuild is in progress, wait for it - if (this.lazyRebuildInProgress && this.lazyRebuildPromise) { - await this.lazyRebuildPromise - return - } - - // Check if lazy rebuild is needed - // Only needed if: disableAutoRebuild=true AND indexes are empty AND storage has data - if (!this.config.disableAutoRebuild) { - // Auto-rebuild is enabled, indexes should already be loaded - return - } - - // Check if storage has data (fast check with limit=1) - const entities = await this.storage.getNouns({ pagination: { limit: 1 } }) - const hasData = (entities.totalCount && entities.totalCount > 0) || entities.items.length > 0 - - if (!hasData) { - // Storage is empty, no rebuild needed - this.lazyRebuildCompleted = true - return - } - - // Start lazy rebuild (with mutex to prevent concurrent rebuilds). - // ALWAYS narrated (prodLog, never the silent-suppressible console): a - // read that triggers an index build must be visible to the operator — - // fifteen silent minutes of a production blackout taught this line. - prodLog.warn( - `[Brainy] first read on this instance is building the derived indexes ` + - `(deferred at open by disableAutoRebuild) — reads WAIT and then serve; ` + - `nothing serves empty. Bounded by store size; progress under [MetadataIndex]/[GraphIndex].` - ) - this.lazyRebuildInProgress = true - this.lazyRebuildPromise = this.rebuildIndexesIfNeeded(true) - .then(() => { - this.lazyRebuildCompleted = true - }) - .finally(() => { - this.lazyRebuildInProgress = false - this.lazyRebuildPromise = null - }) - - await this.lazyRebuildPromise } /** @@ -16392,7 +16317,13 @@ export class Brainy implements BrainyInterface { } /** - * Rebuild indexes from persisted data if needed (LAZY LOADING) + * @description Rebuild indexes from persisted data if needed — THE OPEN-TIME + * BUILD. Called once per open (init calls it; `repairIndex()`'s + * write-quarantine lift calls it forced). Runs to completion BEFORE `init()` + * returns: a needed rebuild is NEVER deferred to a read (there is no more + * first-query lazy path — see {@link ensureIndexesLoaded}, which is a + * read-time CHECK only). `disableAutoRebuild` no longer defers index + * construction to the first query; see its JSDoc in `brainy.types.ts`. * * FIXES FOR CRITICAL BUGS: * - Bug #1: GraphAdjacencyIndex rebuild never called ✅ FIXED @@ -16402,34 +16333,24 @@ export class Brainy implements BrainyInterface { * * Production-grade rebuild with: * - Handles BILLIONS of entities via streaming pagination - * - Smart threshold-based decisions (auto-rebuild < 1000 items) - * - Lazy loading on first query (when disableAutoRebuild: true) + * - A provider's named {@link HealthReport} (when it exposes one) decides + * per-leg need; `isReady()` / a size heuristic decides otherwise — no + * dataset-size threshold gates whether the rebuild runs at open. * - Progress reporting for large datasets * - Parallel index rebuilds for performance * - Robust error recovery (continues on partial failures) - * - Concurrency-safe (multiple queries wait for same rebuild) * - * @param force - Force rebuild even if disableAutoRebuild is true (for lazy loading) + * @param force - Force the rebuild path to run even when no leg reports a need (used by `repairIndex()`'s write-quarantine lift). */ private async rebuildIndexesIfNeeded(force = false): Promise { try { - // Check if auto-rebuild is explicitly disabled (ONLY during init, not for lazy loading) - // force=true means this is a lazy rebuild triggered by first query - if (this.config.disableAutoRebuild === true && !force) { - if (!this.config.silent) { - console.log('⚡ Auto-rebuild explicitly disabled via config') - console.log('💡 Indexes will build automatically on first query (lazy loading)') - } - return - } - // No instant fast-path here: the honest per-leg readiness checks below - // are all O(1) (one bounded storage sample + each provider's size()/ - // isReady()), and this method runs exactly once per open (init calls it; - // the lazy path passes force=true). The removed shortcut keyed off - // `this.index.size() > 0`, a dishonest proxy — it skipped the metadata - // and graph checks whenever the vector happened to be warm, and it never - // fired on a real cold process (the JS vector size is 0 until it loads). + // are all O(1) (one bounded storage sample + each provider's health + // report / size()/isReady()), and this method runs exactly once per + // open. The removed shortcut keyed off `this.index.size() > 0`, a + // dishonest proxy — it skipped the metadata and graph checks whenever + // the vector happened to be warm, and it never fired on a real cold + // process (the JS vector size is 0 until it loads). // BUG #2 FIX: Don't trust counts - check actual storage instead // Counts can be lost/corrupted in container restarts @@ -16448,30 +16369,23 @@ export class Brainy implements BrainyInterface { return } - // Intelligent decision: Auto-rebuild based on dataset size - // Production scale: Handles billions via streaming pagination - const AUTO_REBUILD_THRESHOLD = 10000 // Auto-rebuild if < 10K items (increased from 1K) - // Check if indexes need rebuilding const metadataStats = await this.metadataIndex.getStats() const hnswIndexSize = this.index.size() - // Readiness contract: when a provider exposes isReady(), that honest - // signal REPLACES the size/count heuristic below — an mmap/disk-native - // index legitimately reports 0 resident entries while fully durable on - // disk, and rebuilding it from canonical re-reads every entity file on - // every boot (the 48-seconds-per-restart class a production deployment - // hit). The signal is honest in BOTH directions: a provider whose - // durable state failed to load returns false and gets its rebuild even - // when size() > 0 (the silent-empty cold-load failure). Providers - // without isReady() keep the exact prior empty-heuristics. - const providerReady = (leg: unknown): boolean | undefined => { - const candidate = leg as { isReady?: () => boolean } - return typeof candidate.isReady === 'function' ? candidate.isReady() : undefined + // Readiness contract: a provider's named {@link HealthReport} (when + // exposed) is the authority — `serving === false` needs the rebuild, + // full stop. Absent a health report, fall back to `isReady()` (an + // mmap/disk-native index legitimately reports 0 resident entries while + // fully durable on disk, so rebuilding it from canonical on every boot + // would be the 48-seconds-per-restart class a production deployment + // hit); absent BOTH, keep the per-leg empty-heuristic passed in. + const legNeedsRebuild = (provider: unknown, emptyFallback: boolean): boolean => { + const assessment = assessProviderHealth(provider) + if (assessment.via === 'health-report') return assessment.readiness !== 'ready' + if (assessment.via === 'is-ready') return assessment.readiness === 'not-ready' + return emptyFallback } - const metadataReady = providerReady(this.metadataIndex) - const vectorReady = providerReady(this.index) - const graphReady = providerReady(this.graphIndex) // Epoch-drift trigger: a format-version change makes EVERY derived index // suspect even when each is non-empty, so it forces a rebuild of all @@ -16491,9 +16405,9 @@ export class Brainy implements BrainyInterface { const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating // Per-leg decision, in precedence order: a migrating provider owns its - // index (skip) → epoch drift forces a rebuild → an exposed isReady() - // decides → otherwise a per-leg fallback. The fallbacks differ by leg - // because "empty" means different things: + // index (skip) → epoch drift forces a rebuild → the health-report/ + // isReady() authority decides → otherwise a per-leg fallback. The + // fallbacks differ by leg because "empty" means different things: // - METADATA: past the empty-store early-return, entities exist, so the // id-mapper SHOULD have loaded entries — totalEntries===0 is a real // load-failure signal, so rebuild (self-heal from canonical). @@ -16504,62 +16418,49 @@ export class Brainy implements BrainyInterface { // against canonical) inside storage.getGraphIndex() BEFORE this gate, // so it is already authoritative here; re-deriving would be spurious // (a full O(E) verb scan on every open of an edgeless brain). It - // therefore rebuilds only on epoch drift or a native !isReady(). - // (verifyGraphAdjacencyLive is the query-time backstop.) + // therefore rebuilds only on epoch drift or a native !isReady()/ + // not-serving report. (verifyGraphAdjacencyLive is the query-time + // backstop — it refuses loudly, it never rebuilds.) const shouldRebuildMetadata = !metadataMigrating && - (epochStale || - (metadataReady !== undefined ? !metadataReady : metadataStats.totalEntries === 0)) + (epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0)) const shouldRebuildVector = !vectorMigrating && - (epochStale || (vectorReady !== undefined ? !vectorReady : hnswIndexSize === 0)) + (epochStale || legNeedsRebuild(this.index, hnswIndexSize === 0)) const shouldRebuildGraph = !graphMigrating && - (epochStale || (graphReady !== undefined ? !graphReady : false)) + (epochStale || legNeedsRebuild(this.graphIndex, false)) const needsRebuild = shouldRebuildMetadata || shouldRebuildVector || shouldRebuildGraph if (!needsRebuild && !force) { - // All indexes report current — durably loaded (isReady/size), or owned - // by a background migration. No rebuild needed. + // All indexes report current — durably loaded (health-report/isReady/ + // size), or owned by a background migration. No rebuild needed. return } - // Determine rebuild strategy - const isLazyRebuild = force && this.config.disableAutoRebuild === true - const isSmallDataset = totalCount < AUTO_REBUILD_THRESHOLD - const shouldRebuild = isLazyRebuild || isSmallDataset || this.config.disableAutoRebuild === false + // Name exactly which legs rebuild — "all indexes" was a lie whenever + // the durable legs were skipped (e.g. only the JS vector index loads + // here on a warm reopen), and it misread as a whole-brain rebuild in + // consumer boot logs. + const rebuildingLegs = [ + shouldRebuildMetadata && 'metadata', + shouldRebuildVector && 'vector', + shouldRebuildGraph && 'graph' + ] + .filter(Boolean) + .join(' + ') - if (!shouldRebuild) { - // Large dataset with auto-rebuild disabled: Wait for lazy loading - if (!this.config.silent) { - console.log(`⚡ Large dataset (${totalCount.toLocaleString()} items) - using lazy loading for optimal startup`) - console.log('💡 Indexes will build automatically on first query') - } - return - } - - // REBUILD: Either small dataset, forced rebuild, or explicit enable - const rebuildReason = isLazyRebuild - ? '🔄 Lazy loading triggered by first query' - : isSmallDataset - ? `🔄 Small dataset (${totalCount.toLocaleString()} items)` - : '🔄 Auto-rebuild explicitly enabled' - - if (!this.config.silent) { - // Name exactly which legs rebuild — "all indexes" was a lie whenever - // the durable legs were skipped (e.g. only the JS vector index loads - // here on a warm reopen), and it misread as a whole-brain rebuild in - // consumer boot logs. - const rebuildingLegs = [ - shouldRebuildMetadata && 'metadata', - shouldRebuildVector && 'vector', - shouldRebuildGraph && 'graph' - ] - .filter(Boolean) - .join(' + ') - console.log(`${rebuildReason} - loading/rebuilding ${rebuildingLegs || 'no'} index(es) from persisted data...`) - } + // ALWAYS narrated (prodLog, never the silent-suppressible console): there + // is no more first-query lazy path — a rebuild that runs here BLOCKS + // open() regardless of dataset size or `disableAutoRebuild`, so an + // operator must see it in the boot log, not discover it as an + // unexplained slow open. + prodLog.warn( + `[Brainy] open() is building/rebuilding the ${rebuildingLegs || 'no'} index(es) from ` + + `${totalCount.toLocaleString()} stored entities — open blocks until the derived ` + + `indexes serve; reads never build.` + ) // Before the graph rebuild, hydrate the entity id-mapper from the persisted // snapshot. A native int-keyed adjacency resolves every verb endpoint through @@ -16586,13 +16487,23 @@ export class Brainy implements BrainyInterface { const rebuildDuration = Date.now() - rebuildStartTime const metadataCountAfter = (await this.metadataIndex.getStats()).totalEntries + const graphSizeAfter = await this.graphIndex.size() + + // Completion narration — ALWAYS via prodLog (see the pre-rebuild narration + // above for why): the operator who saw "open() is building…" needs the + // matching "…and it's done" line, with the numbers to confirm it worked. + prodLog.warn( + `[Brainy] open() finished building derived indexes in ${rebuildDuration}ms: ` + + `metadata=${metadataCountAfter} entries, vector=${this.index.size()} nodes, ` + + `graph=${graphSizeAfter} relationships.` + ) if (!this.config.silent) { console.log( `All indexes rebuilt in ${rebuildDuration}ms:\n` + ` - Metadata: ${metadataCountAfter} entries\n` + ` - HNSW Vector: ${this.index.size()} nodes\n` + - ` - Graph Adjacency: ${await this.graphIndex.size()} relationships` + ` - Graph Adjacency: ${graphSizeAfter} relationships` ) } @@ -16998,49 +16909,6 @@ export class Brainy implements BrainyInterface { return result } - /** - * Run the optional metadata cold-open consistency probe at most once per brain. - * When the active provider exposes `probeConsistency()` (the native cross-bucket - * O(1) sampler), a `false` result triggers `detectAndRepairCorruption()` so an - * already-poisoned index self-heals on first read — the metadata counterpart of - * the 7.33.2 graph cold-load guard. Best-effort: a probe failure never breaks the - * read (the guard is reset so a transient failure retries). No-op for the JS index - * (it exposes no probe), and the full-scan `validateConsistency` stays the explicit - * deep diagnostic via `validateIndexConsistency()`. - */ - private async ensureMetadataConsistencyProbed(): Promise { - if (this._metadataConsistencyProbed) return - // Defer while the metadata provider runs its one-time in-place migration: - // probing (and self-healing via rebuild) an index the provider is mid-rebuild - // would collide with the provider that owns it. Mirrors the vector deference - // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next - // read runs the probe. (The family-scoped find() gate waits on the metadata - // family separately before any actual filter read.) - if (this.providerIsMigrating(this.metadataIndex)) return - this._metadataConsistencyProbed = true - const provider = this.metadataIndex as { - probeConsistency?: () => Promise - detectAndRepairCorruption?: () => Promise - } - if (typeof provider.probeConsistency !== 'function') return - try { - const healthy = await provider.probeConsistency() - if (!healthy && typeof provider.detectAndRepairCorruption === 'function') { - if (!this.config.silent) { - console.warn('[Brainy] metadata index failed the cold-open consistency probe — self-healing via rebuild.') - } - await provider.detectAndRepairCorruption() - } - } catch (error) { - // The self-heal is best-effort and must never break a read. Reset the guard - // so a transient probe failure is retried on the next read. - this._metadataConsistencyProbed = false - if (!this.config.silent) { - console.warn('[Brainy] metadata cold-open consistency probe failed (continuing):', error) - } - } - } - /** * Detect and repair corrupted metadata indexes. * @@ -17166,7 +17034,28 @@ export class Brainy implements BrainyInterface { ) } - async repairIndex(): Promise { + /** + * @description The ceremony door for index repair. Bare `repairIndex()` is + * REPORT-DRIVEN, exactly as before: it prunes orphans, recomputes count + * rollups, reconciles VFS containment, and — for the three derived-index + * providers — consults each one's `validateInvariants()` and rebuilds only + * a family whose failing invariant asks for it (`heal: 'rebuild'`). + * + * `options.rebuild` is the EXPLICIT operator override: name one or more + * families (or `'all'`) to rebuild them UNCONDITIONALLY — no invariant is + * consulted, JS or native provider alike. Use it when an operator has + * independent reason to believe a family needs reconciling regardless of + * what its own self-report says (a report can only be as honest as the + * provider that produced it). A family named here is recorded as its own + * `provider:` row with `rebuilt: true` and + * `reason: 'explicit rebuild requested'`, and is SKIPPED by the normal + * invariant-driven pass (it was already rebuilt unconditionally — a second, + * report-driven pass over the same family would be redundant at best). + * + * @param options.rebuild - Family name(s) to unconditionally rebuild, or `'all'` for all three (`'metadata' | 'graph' | 'vector'`). + * @returns The full per-family receipt (see {@link RepairReport}); also narrated via `prodLog.warn`. + */ + async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { await this.ensureInitialized() const startedAt = Date.now() const families: RepairFamilyReport[] = [] @@ -17271,17 +17160,52 @@ export class Brainy implements BrainyInterface { `Writes are re-enabled.` ) } + // THE CEREMONY DOOR: an explicit `options.rebuild` names a family (or + // 'all') to rebuild UNCONDITIONALLY — no invariant consulted. Resolved + // here so the loop below can skip a family's normal report-driven pass + // once its unconditional rebuild has already run. + const explicitRebuildFamilies: ReadonlySet<'metadata' | 'vector' | 'graph'> = + options?.rebuild === 'all' + ? new Set<'metadata' | 'vector' | 'graph'>(['metadata', 'vector', 'graph']) + : new Set(options?.rebuild ?? []) + // Cross-layer repair: repairIndex must reconcile NATIVE derived // state from canonical, not just the JS metadata index. Consult each provider's // own validateInvariants() and rebuild any whose failing invariant asks for it // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption(). - for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { + const providerFamilies: ReadonlyArray = [ + ['metadata', this.metadataIndex], + ['vector', this.index], + ['graph', this.graphIndex] + ] + for (const [familyName, provider] of providerFamilies) { + if (explicitRebuildFamilies.has(familyName)) { + const p = provider as { rebuild?: () => Promise } | null + if (!p || typeof p.rebuild !== 'function') { + record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no rebuild() contract' }) + continue + } + prodLog.warn( + `[Brainy] repairIndex(): explicit rebuild requested for '${familyName}' — ` + + `rebuilding unconditionally (no invariant consulted).` + ) + await p.rebuild() + record(`provider:${familyName}`, { + checked: true, + healed: 1, + rebuilt: true, + reason: 'explicit rebuild requested' + }) + prodLog.warn(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) + continue + } + const p = provider as { validateInvariants?: () => Promise rebuild?: () => Promise } | null if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') { - record(`provider:${(provider as { constructor?: { name?: string } })?.constructor?.name ?? 'unknown'}`, { + record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract' }) continue @@ -17290,7 +17214,7 @@ export class Brainy implements BrainyInterface { try { report = await p.validateInvariants() } catch (err) { - record(`provider:unknown`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` }) + record(`provider:${familyName}`, { checked: false, healed: 0, skipped: `validateInvariants threw: ${(err as Error).message}` }) continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here } if (report.healthy) { diff --git a/src/index.ts b/src/index.ts index 765c20b4..07f318c0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -269,6 +269,10 @@ export type { FamilyStamp, StampMembers, StampVerdict } from './db/familyStamp.j export { isVersionedIndexProvider } from './plugin.js' export type { VersionedIndexProvider } from './plugin.js' export type { ProviderInvariantReport, InvariantResult, InvariantHeal } from './plugin.js' +// The named, synchronous, O(1) health-report contract (the read gate's ONLY +// source of truth for "can I serve right now") — see HealthReport's +// derivation laws in plugin.ts. +export type { HealthReport, LedgerInvariantResult, InvariantSource } from './plugin.js' // Optional provider self-report of outstanding background maintenance work // (compaction, deferred writes, etc.) — the payload type for // brain.maintenanceDebt(). See the measure-only-what-you-track contract on diff --git a/src/plugin.ts b/src/plugin.ts index 947c86a5..47a559b6 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -171,6 +171,66 @@ export interface ProviderInvariantReport { durationMs: number } +/** + * @description Where a {@link LedgerInvariantResult} verdict came from: + * - `'ledger'` — decided from an exact, durable ledger (a real count, not a sample). + * - `'deep'` — decided by a full/expensive scan (the `validateInvariants()` diagnostic path only). + * - `'unledgered'` — this family has no ledger yet; the verdict is UNKNOWN, never healthy and never broken. + */ +export type InvariantSource = 'ledger' | 'deep' | 'unledgered' + +/** + * @description One invariant verdict inside a {@link HealthReport}. Extends + * {@link InvariantResult} with the provenance of the verdict ({@link InvariantSource}) + * and, for a failing set-membership invariant, an exact count plus a capped sample + * of the diverging ids — a VERDICT, never a dump. `sample` MUST be capped at 16 ids; + * `count` is the exact number even when `sample` is truncated. + */ +export interface LedgerInvariantResult extends InvariantResult { + /** Provenance of this verdict — see {@link InvariantSource}. */ + source: InvariantSource + /** Exact count of diverging/missing items plus a capped (≤16 ids) sample. Present only on a failing set-membership invariant. */ + missing?: { count: number; sample: string[] } +} + +/** + * @description The NAMED, SYNCHRONOUS, O(1) health report a provider exposes via + * {@link MetadataIndexProvider.healthReport} / {@link GraphIndexProvider.healthReport} / + * {@link VectorIndexProvider.healthReport}. This is the read gate's ONLY source of + * truth for "can I serve right now" — it replaces sampled self-probes and the + * unnamed `isReady()` latch with an exact, ledger-derived verdict. + * + * Derivation laws (a provider MUST honor these; brainy's read gate assumes them): + * - `healthy` = every VERIFIED invariant in {@link invariants} holds. An invariant + * whose family is named in {@link unledgered} is NEVER counted toward `healthy` + * either way — it is unknown, not passing. + * - `serving` = no verified invariant in {@link invariants} FAILS with `heal: 'rebuild'`. + * A failure with `heal: 'repair'` or `heal: 'none'` is degraded-but-serving — + * `serving` stays `true`. Only a `'rebuild'`-grade failure makes `serving` `false`. + * - `validateInvariants()` remains the async DEEP diagnostic (full scans allowed, + * `source: 'deep'` results); `healthReport()` MUST be synchronous, O(1) from + * exact ledgers/counters, and MUST NOT throw for a well-formed provider — a + * provider that cannot produce a safe verdict reports it as a failing invariant, + * it does not throw (a throw is read by the gate as a CONTRACT VIOLATION, not as + * "unknown"). + */ +export interface HealthReport extends ProviderInvariantReport { + /** + * Monotonic per provider: bumps on every ledger mutation and every rebuild + * boundary. Consumers (the read gate's narration dedup, external callers) may + * cache a verdict per generation. + */ + generation: number + /** Each checked invariant, with provenance — see {@link LedgerInvariantResult}. */ + invariants: LedgerInvariantResult[] + /** + * Families with no ledger yet. NAMED here so an operator can see what is not + * yet tracked — NEVER counted as healthy (they are not verified) and NEVER + * counted as broken (there is nothing to fail). + */ + unledgered: string[] +} + /** * @description A provider's self-report of its own outstanding background * maintenance work (compaction, deferred writes, a build-new→verify→swap in @@ -266,6 +326,20 @@ export interface MetadataIndexProvider { */ validateInvariants?(): Promise + /** + * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this + * provider derives from its own exact ledgers — see {@link HealthReport} for + * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a + * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never + * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) + * reads THIS instead of `isReady()` / size heuristics: `serving` decides + * whether reads may proceed; a `false` refuses the read loudly rather than + * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the + * size heuristic (this train's JS built-in providers stay on that interim + * path). + */ + healthReport?(): HealthReport + /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background @@ -462,6 +536,20 @@ export interface GraphIndexProvider { */ validateInvariants?(): Promise + /** + * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this + * provider derives from its own exact ledgers — see {@link HealthReport} for + * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a + * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never + * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) + * reads THIS instead of `isReady()` / size heuristics: `serving` decides + * whether reads may proceed; a `false` refuses the read loudly rather than + * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the + * size heuristic (this train's JS built-in providers stay on that interim + * path). + */ + healthReport?(): HealthReport + /** * @description OPTIONAL eager cold-load. Called once during brain init — AFTER * the metadata provider's `init()` (so the id-mapper is hydrated; a native int @@ -1225,6 +1313,20 @@ export interface VectorIndexProvider { */ validateInvariants?(): Promise + /** + * @description OPTIONAL. The named, SYNCHRONOUS, O(1) health verdict this + * provider derives from its own exact ledgers — see {@link HealthReport} for + * the full derivation laws. MUST NOT perform I/O and MUST NOT throw for a + * well-formed provider (brainy treats a throw as a CONTRACT VIOLATION, never + * as "unknown"). When present, brainy's read gate (`assessProviderHealth()`) + * reads THIS instead of `isReady()` / size heuristics: `serving` decides + * whether reads may proceed; a `false` refuses the read loudly rather than + * triggering a rebuild. Absent → the gate falls back to `isReady?()` / the + * size heuristic (this train's JS built-in providers stay on that interim + * path). + */ + healthReport?(): HealthReport + /** * @description OPTIONAL. A native provider returns true from the moment its * `init()` detects a large epoch-drift until its background diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 78e310f3..38bc4aa6 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1816,10 +1816,16 @@ export interface BrainyConfig { | StorageAdapter /** - * Disable the automatic index rebuild check during `init()`. By default - * Brainy auto-decides from dataset size: small datasets rebuild missing - * indexes inline, large datasets rebuild lazily on first query. Set `true` - * only when an operator wants full manual control via `repairIndex()`. + * RE-MEANT (the health-gate contract): `init()` (open) always verifies the + * durable generation of every derived index, and a needed rebuild ALWAYS + * runs at open — it is never deferred to the first read, regardless of + * dataset size or this flag. There is no first-query lazy-build path + * anymore: a read that finds a provider not serving throws a typed + * `*NotReadyError` rather than building anything (see + * `assessProviderHealth` / the read gate in `brainy.ts`). Setting this + * `true` no longer defers index construction to the first query — it has + * no effect on WHEN a needed rebuild runs. Full manual control over + * rebuilds remains available via `repairIndex({ rebuild: [...] })`. */ disableAutoRebuild?: boolean diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts index 16266bec..498f2003 100644 --- a/src/utils/indexReadiness.ts +++ b/src/utils/indexReadiness.ts @@ -13,13 +13,28 @@ * `size()` or `isInitialized`. When `isReady()` is absent, callers must fall back * to a KNOWN-ITEM PROBE (a real search/lookup that must return a known-present * datum) before trusting an empty result — never a `size()` proxy. + * + * {@link assessProviderHealth} is the NEWER, PREFERRED authority: it reads a + * provider's NAMED, synchronous, O(1) {@link import('../plugin.js').HealthReport} + * when one is exposed, and falls back to this file's `isReady()` classifier only + * when the provider does not (yet) expose a health report. Read paths in + * `brainy.ts` call `assessProviderHealth` exclusively — `assessIndexReadiness` + * stays exported for the other call sites (`storage/baseStorage.ts`) and for the + * fallback branch inside `assessProviderHealth` itself. */ +import type { HealthReport } from '../plugin.js' + /** A provider that MAY expose the honest cold-load readiness signal. */ export interface MaybeReadyProvider { isReady?: () => boolean } +/** A provider that MAY expose the named, synchronous, O(1) health report. */ +export interface MaybeHealthReportingProvider { + healthReport?: () => HealthReport +} + /** Three-valued honest-readiness verdict. */ export type IndexReadiness = 'ready' | 'not-ready' | 'unknown' @@ -36,3 +51,105 @@ export function assessIndexReadiness(provider: unknown): IndexReadiness { if (p == null || typeof p.isReady !== 'function') return 'unknown' return p.isReady() ? 'ready' : 'not-ready' } + +/** + * @description Which signal {@link assessProviderHealth} actually consulted to + * produce its verdict — surfaced so callers can narrate (and tests can pin) how + * a provider was judged, not just what the judgment was. + * - `'health-report'` — the provider's `healthReport()` was called (the authority). + * - `'is-ready'` — no `healthReport()`; fell back to the provider's `isReady()`. + * - `'size-heuristic'` — no `healthReport()` and no `isReady()`; caller must keep its own size-based heuristic. + * - `'none'` — there was no provider to assess (`null`/`undefined`). + */ +export type ProviderHealthVia = 'health-report' | 'is-ready' | 'size-heuristic' | 'none' + +/** The result of {@link assessProviderHealth}. */ +export interface ProviderHealthAssessment { + /** The honest readiness verdict — see {@link IndexReadiness}. */ + readiness: IndexReadiness + /** The provider's raw {@link HealthReport}, when one was obtained; `null` otherwise. */ + report: HealthReport | null + /** Which signal produced the verdict — see {@link ProviderHealthVia}. */ + via: ProviderHealthVia + /** Human-readable reasons: named failing invariants (with `heal`), unledgered families, or the fallback-path explanation. Empty when the provider is healthy and ready. */ + reasons: string[] +} + +/** + * @description THE read-gate authority. Prefers a provider's NAMED, + * synchronous, O(1) {@link HealthReport} over the older `isReady()` / size + * heuristics; falls back to {@link assessIndexReadiness}'s semantics only when + * a provider does not (yet) expose `healthReport()`. + * + * Derivation: + * - `healthReport()` present → call it (wrapped in try/catch). A THROW is a + * CONTRACT VIOLATION, not "unknown": returns `readiness: 'not-ready'`, + * `via: 'health-report'`, and a reason naming the throw — never swallowed + * into `'unknown'`. + * - Otherwise → `readiness = report.serving ? 'ready' : 'not-ready'`; `reasons` + * names every invariant with `holds: false` (with its `heal`), plus an + * `unledgered: [...]` line when {@link HealthReport.unledgered} is non-empty. + * UNLEDGERED IS UNKNOWN: an unledgered family never flips a serving provider + * to not-ready, and never flips a not-serving provider to ready — `serving` + * is always the provider's own verdict, verbatim. + * - No `healthReport()` → fall back to {@link assessIndexReadiness}'s semantics: + * `via: 'is-ready'` when `isReady()` exists, `via: 'size-heuristic'` when + * neither hook exists (caller must keep its own size-based heuristic), + * `via: 'none'` when there is no provider at all. + * @param provider - Any index provider (vector / graph / metadata) or `null`/`undefined`. + */ +export function assessProviderHealth(provider: unknown): ProviderHealthAssessment { + const p = provider as (MaybeHealthReportingProvider & MaybeReadyProvider) | null | undefined + + if (p == null) { + return { readiness: 'unknown', report: null, via: 'none', reasons: ['no provider to assess'] } + } + + if (typeof p.healthReport === 'function') { + let report: HealthReport + try { + report = p.healthReport() + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + return { + readiness: 'not-ready', + report: null, + via: 'health-report', + reasons: [`healthReport() threw: ${message} — a health-report throw is a contract violation, never read as healthy`] + } + } + + const reasons: string[] = [] + for (const invariant of report.invariants) { + if (!invariant.holds) { + reasons.push(`${invariant.name} (heal:${invariant.heal}): ${invariant.detail}`) + } + } + if (report.unledgered.length > 0) { + reasons.push(`unledgered: ${report.unledgered.join(', ')}`) + } + + return { + readiness: report.serving ? 'ready' : 'not-ready', + report, + via: 'health-report', + reasons + } + } + + const readiness = assessIndexReadiness(p) + if (readiness === 'unknown') { + return { + readiness, + report: null, + via: 'size-heuristic', + reasons: ['provider exposes neither healthReport() nor isReady() — falling back to the size heuristic'] + } + } + return { + readiness, + report: null, + via: 'is-ready', + reasons: readiness === 'not-ready' ? ['isReady() returned false'] : [] + } +} diff --git a/tests/configs/vitest.integration.config.ts b/tests/configs/vitest.integration.config.ts index 3d3a3721..af86097d 100644 --- a/tests/configs/vitest.integration.config.ts +++ b/tests/configs/vitest.integration.config.ts @@ -20,6 +20,9 @@ export default defineConfig({ // Include only integration tests include: [ 'tests/integration/**/*.test.ts', + // The lifecycle biography lane (day-in-the-life scenarios; see + // tests/lifecycle/README.md) runs in the integration gate. + 'tests/lifecycle/**/*.test.ts', 'tests/**/*.integration.test.ts' ], diff --git a/tests/integration/cold-graph-connected-8.0.test.ts b/tests/integration/cold-graph-connected-8.0.test.ts index 71ae345d..7ee725f5 100644 --- a/tests/integration/cold-graph-connected-8.0.test.ts +++ b/tests/integration/cold-graph-connected-8.0.test.ts @@ -1,29 +1,29 @@ /** * @module tests/integration/cold-graph-connected-8.0 * @description BRAINY-COLD-GRAPH-CONNECTED (8.0) — regression coverage for the silent-empty - * graph-traversal bug, gated on the converged 8.0 contract: a sync `graphIndex.isReady()` that - * is true ONLY when the source→target EDGES are loaded (NOT the membership/manifest count). + * graph-traversal bug, gated on the honest readiness signal: a sync `graphIndex.isReady()` + * that is true ONLY when the source→target EDGES are loaded (NOT the membership/manifest count). * - * On the FIRST `find({ connected })` after a cold process start of a LARGE brain (≥10k nouns, - * which skips the eager index rebuild), a native graph adjacency can reload its relationship - * COUNT (so `size() > 0`) but NOT its edges — so `getNeighbors()` returns `[]` for EVERY source - * and brainy would serve that `[]` as if the anchor were genuinely edgeless. + * On the FIRST `find({ connected })` after a cold process start, a native graph adjacency can + * reload its relationship COUNT (so `size() > 0`) but NOT its edges — so `getNeighbors()` returns + * `[]` for EVERY source and brainy would serve that `[]` as if the anchor were genuinely edgeless. * - * The 8.0 guard (`verifyGraphAdjacencyLive`) prefers the honest `isReady()` signal: - * - `isReady() === false` → hydrate the id-mapper, rebuild from storage, re-check; a still-false - * `isReady()` throws {@link GraphIndexNotReadyError} instead of returning `[]` ('rebuilt' when - * the rebuild heals it); + * RE-POINTED to the health-gate law: `verifyGraphAdjacencyLive` NEVER rebuilds and NEVER walks the + * store from a read — a read-path rebuild is exactly the dark-rebuild failure mode the law retires + * (open() alone owns building). The guard now: + * - `isReady() === false` → THROWS {@link GraphIndexNotReadyError} immediately — no rebuild attempt; * - a genuinely edgeless anchor with `isReady() === true` verifies 'live' and the empty result - * stands — no spurious rebuild, no throw; - * - a provider WITHOUT `isReady()` falls back to the shipped 7.x known-edge-sample probe. + * stands — no spurious throw; + * - a provider WITHOUT `isReady()` falls back to the shipped known-edge-sample probe, which is + * now READ-ONLY: it refuses loudly (throws) rather than self-healing via rebuild. * * These exercise REAL `find({ connected })` against an in-memory brain whose graph index is - * instrumented with a test-double `isReady()` (and, for the fallback case, an empty-then-healed + * instrumented with a test-double `isReady()` (and, for the fallback case, an always-empty * `getNeighbors`). Only the readiness/edge surface is wrapped; the underlying real adjacency - * (built by `relate()`) is unmasked once a rebuild "heals" it. + * (built by `relate()`) is what a healthy provider actually serves. */ -import { describe, it, expect, afterEach } from 'vitest' +import { describe, it, expect, afterEach, vi } from 'vitest' import { Brainy } from '../../src/index.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' import { GraphIndexNotReadyError } from '../../src/errors/brainyError.js' @@ -63,17 +63,17 @@ async function buildBrain( } /** - * Instrument the brain's real graph index with a test-double `isReady()` (the 8.0 contract) plus - * an edge surface that goes empty while NOT ready. `getNeighbors` returns `[]` while `!ready` - * (modelling the cold-unloaded adjacency) and delegates to the REAL index once a rebuild flips - * `ready` on. `rebuild` is counted; it heals (`ready = true`) only when `healsOnRebuild` is set. - * Pass `failFirstRebuild` to make the FIRST rebuild throw a transient error (without healing) so - * the empty-result re-collect path in executeGraphSearch is exercised. + * Instrument the brain's real graph index with a test-double `isReady()` (the honest-readiness + * contract) plus an edge surface that goes empty while NOT ready. `getNeighbors` returns `[]` + * while `!ready` (modelling the cold-unloaded adjacency) and delegates to the REAL index once + * `ready` flips true (used only by the "healthy" control cases — the guard itself never flips + * this anymore, since it never rebuilds). `rebuild` is counted so tests can assert it is NEVER + * called by a read. */ function instrumentIsReady( brain: any, - opts: { ready: boolean; healsOnRebuild: boolean; failFirstRebuild?: boolean } -): { rebuildCalls: number } { + opts: { ready: boolean } +): { rebuildCalls: number; ready: boolean } { const gi = brain.graphIndex const origGetNeighbors = gi.getNeighbors.bind(gi) const state = { ready: opts.ready, rebuildCalls: 0 } @@ -85,10 +85,6 @@ function instrumentIsReady( gi.rebuild = async (): Promise => { state.rebuildCalls++ - if (opts.failFirstRebuild && state.rebuildCalls === 1) { - throw new Error('transient rebuild hiccup') - } - if (opts.healsOnRebuild) state.ready = true // unmask the real (already-populated) adjacency } return state @@ -96,12 +92,12 @@ function instrumentIsReady( /** * Fallback instrumentation — a provider WITHOUT `isReady()` (older cortex / JS baseline). Wraps - * `getNeighbors` to return `[]` while `broken` and delegates to the REAL index once a rebuild - * heals it. This is the shipped 7.x known-edge-sample probe path on 8.0. + * `getNeighbors` to always return `[]` while `broken`. This is the shipped known-edge-sample + * probe path — now READ-ONLY: it refuses loudly rather than self-healing. */ function instrumentNoIsReady( brain: any, - opts: { broken: boolean; healsOnRebuild: boolean } + opts: { broken: boolean } ): { rebuildCalls: number } { const gi = brain.graphIndex // Ensure the provider does NOT expose isReady() — the default JS provider doesn't. @@ -114,13 +110,12 @@ function instrumentNoIsReady( gi.rebuild = async (): Promise => { state.rebuildCalls++ - if (opts.healsOnRebuild) state.broken = false } return state } -describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a silent []', () => { +describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a silent [], never rebuilds from a read', () => { let brains: any[] = [] afterEach(async () => { for (const b of brains) { @@ -131,35 +126,37 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si } } brains = [] + vi.restoreAllMocks() }) - it('(a) isReady() false → rebuild heals it true → find({ connected }) returns correct N (rebuilt)', async () => { - const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) - brains.push(brain) - const state = instrumentIsReady(brain, { ready: false, healsOnRebuild: true }) - - const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) - - expect(state.rebuildCalls).toBeGreaterThanOrEqual(1) // detected not-ready + healed it - const ids = results.map((r: any) => r.id).sort() - expect(ids).toEqual(targetIds.sort()) // B, C, D — the real edges, served after the heal - }) - - it('(b) isReady() stays false after rebuild → throws GraphIndexNotReadyError (NOT a silent [])', async () => { + it('(a) isReady() false → THROWS GraphIndexNotReadyError immediately, no rebuild attempt', async () => { const { brain, anchorId } = await buildBrain({ anchorEdges: true }) brains.push(brain) - instrumentIsReady(brain, { ready: false, healsOnRebuild: false }) // rebuild never makes it ready + const state = instrumentIsReady(brain, { ready: false }) await expect( brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) ).rejects.toBeInstanceOf(GraphIndexNotReadyError) + + expect(state.rebuildCalls).toBe(0) // a read never rebuilds — it refuses loudly instead + }) + + it('(b) isReady() stays false → throws GraphIndexNotReadyError (NOT a silent [])', async () => { + const { brain, anchorId } = await buildBrain({ anchorEdges: true }) + brains.push(brain) + const state = instrumentIsReady(brain, { ready: false }) + + await expect( + brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + ).rejects.toBeInstanceOf(GraphIndexNotReadyError) + expect(state.rebuildCalls).toBe(0) }) it('(c) edgeless anchor + isReady() true → returns [] with NO rebuild and NO throw', async () => { // The anchor has no edges, but E -> F does — the adjacency is genuinely loaded (ready). const { brain, anchorId } = await buildBrain({ anchorEdges: false }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: true, healsOnRebuild: false }) + const state = instrumentIsReady(brain, { ready: true }) const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) @@ -170,7 +167,7 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si it('(d) healthy isReady() true → correct results, NO rebuild', async () => { const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: true, healsOnRebuild: false }) + const state = instrumentIsReady(brain, { ready: true }) const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) @@ -179,30 +176,30 @@ describe('BRAINY-COLD-GRAPH-CONNECTED 8.0 — isReady()-gated, never serves a si expect(ids).toEqual(targetIds.sort()) }) - it('(e) provider WITHOUT isReady() → falls back to the known-edge-sample probe (self-heals)', async () => { - const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + it('(e) provider WITHOUT isReady() → the known-edge-sample probe REFUSES LOUDLY (never self-heals)', async () => { + const { brain, anchorId } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentNoIsReady(brain, { broken: true, healsOnRebuild: true }) + const state = instrumentNoIsReady(brain, { broken: true }) - const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + await expect( + brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + ).rejects.toBeInstanceOf(GraphIndexNotReadyError) - expect(state.rebuildCalls).toBeGreaterThanOrEqual(1) // detected the empty adjacency + healed it - const ids = results.map((r: any) => r.id).sort() - expect(ids).toEqual(targetIds.sort()) // B, C, D — served after the heal + expect(state.rebuildCalls).toBe(0) // the fallback probe is READ-ONLY — it never calls rebuild() }) - it('(f) executeGraphSearch re-collect: a transient first rebuild leaves connectedIds empty; the empty-result guard then heals + re-collects', async () => { - // First verify (inside neighbors()) hits a transient rebuild failure → returns 'live' without - // healing, so getNeighbors stays empty and connectedIds is empty. The empty connectedIds set - // then drives executeGraphSearch's own verify, whose rebuild now heals → 'rebuilt' → re-collect. - const { brain, anchorId, targetIds } = await buildBrain({ anchorEdges: true }) + it('(f) an empty connectedIds set re-verifies against a not-serving adjacency and throws, rather than serving [] as truth', async () => { + // executeGraphSearch's cold-load guard (connectedIds.size === 0 → re-verify) used to + // interpret a healed rebuild as "re-collect and serve." That rebuild-and-heal path is + // retired: the re-verify now either confirms a genuinely edgeless anchor ('live', case (c)) + // or — as here — discovers the adjacency itself is not serving, and throws. + const { brain, anchorId } = await buildBrain({ anchorEdges: true }) brains.push(brain) - const state = instrumentIsReady(brain, { ready: false, healsOnRebuild: true, failFirstRebuild: true }) + const state = instrumentIsReady(brain, { ready: false }) - const results = await brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) - - expect(state.rebuildCalls).toBeGreaterThanOrEqual(2) // first transient, second heals - const ids = results.map((r: any) => r.id).sort() - expect(ids).toEqual(targetIds.sort()) // re-collected after the heal + await expect( + brain.find({ connected: { from: anchorId, direction: 'out' }, limit: 10 }) + ).rejects.toBeInstanceOf(GraphIndexNotReadyError) + expect(state.rebuildCalls).toBe(0) }) }) diff --git a/tests/integration/health-gate.test.ts b/tests/integration/health-gate.test.ts new file mode 100644 index 00000000..4c4fb454 --- /dev/null +++ b/tests/integration/health-gate.test.ts @@ -0,0 +1,352 @@ +/** + * @module tests/integration/health-gate + * @description Pins for the health-by-accounting read gate: the read gate stops + * consulting an unnamed `isReady()` boolean and reads a NAMED, sync, O(1) + * {@link HealthReport}; no read path may ever start a store walk; the open path + * brings every provider to serving before it returns; an explicit operator door + * (`repairIndex({ rebuild: [...] })`) rebuilds a named leg unconditionally. + * + * Providers here are white-box test doubles: a `healthReport()` (or, for the + * interim-path pins, an `isReady()`) function assigned directly onto the LIVE + * JS provider object, the same pattern `tests/unit/validate-invariants-delegation.test.ts` + * uses for `validateInvariants`. This exercises brainy's real gate/verify code + * against a controlled provider self-report — no engine mocks. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + Brainy, + NounType, + VerbType, + GraphIndexNotReadyError, + MetadataIndexNotReadyError, + VectorIndexNotReadyError +} from '../../src/index.js' +import type { HealthReport, LedgerInvariantResult } from '../../src/plugin.js' +import { prodLog } from '../../src/utils/logger.js' +import { createTestConfig } from '../helpers/test-factory.js' + +/** The white-box surface these pins drive on a live brain instance. */ +interface BrainInternals { + storage: { + getNoun(id: string): Promise + getNounMetadata(id: string): Promise + getNouns(options?: unknown): Promise + getVerbs(options?: unknown): Promise + } + index: { healthReport?: () => HealthReport; isReady?: () => boolean; rebuild(): Promise } + metadataIndex: { + healthReport?: () => HealthReport + isReady?: () => boolean + rebuild(): Promise + validateInvariants?: () => Promise + } + graphIndex: { + healthReport?: () => HealthReport + isReady?: () => boolean + rebuild(): Promise + validateInvariants?: () => Promise + } + rebuildIndexesIfNeeded(force?: boolean): Promise +} + +function internalsOf(brain: Brainy): BrainInternals { + return brain as unknown as BrainInternals +} + +function invariant(overrides: Partial = {}): LedgerInvariantResult { + return { + name: 'manifest-residency', + holds: true, + detail: 'ok', + heal: 'none', + source: 'ledger', + ...overrides + } +} + +function healthReport(overrides: Partial = {}): HealthReport { + return { + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 1, + generation: 1, + unledgered: [], + ...overrides + } +} + +const brains: Brainy[] = [] +const dirs: string[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) + vi.restoreAllMocks() +}) + +describe('health gate (a) — not-serving refuses loudly, ZERO canonical reads during the refusal', () => { + it('metadata not-serving: find() throws MetadataIndexNotReadyError naming the failing invariant', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + internals.metadataIndex.healthReport = () => + healthReport({ + provider: 'metadata', + serving: false, + healthy: false, + invariants: [invariant({ name: 'posted-count-floor', holds: false, heal: 'rebuild', detail: 'posted 2 < canonical 5' })] + }) + + const getNounSpy = vi.spyOn(internals.storage, 'getNoun') + const getNounMetadataSpy = vi.spyOn(internals.storage, 'getNounMetadata') + const getNounsSpy = vi.spyOn(internals.storage, 'getNouns') + + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError) + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/posted-count-floor/) + + expect(getNounSpy).not.toHaveBeenCalled() + expect(getNounMetadataSpy).not.toHaveBeenCalled() + expect(getNounsSpy).not.toHaveBeenCalled() + + delete internals.metadataIndex.healthReport + }) + + it('graph not-serving: related() throws GraphIndexNotReadyError naming the failing invariant, no canonical reads', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + const a = await brain.add({ data: 'a', type: NounType.Person }) + const b = await brain.add({ data: 'b', type: NounType.Person }) + await brain.relate({ from: a, to: b, type: VerbType.Knows }) + await brain.flush() + + const internals = internalsOf(brain) + internals.graphIndex.healthReport = () => + healthReport({ + provider: 'graph', + serving: false, + healthy: false, + invariants: [invariant({ name: 'adjacency-residency', holds: false, heal: 'rebuild', detail: 'edges not loaded' })] + }) + + const getNounSpy = vi.spyOn(internals.storage, 'getNoun') + const getVerbsSpy = vi.spyOn(internals.storage, 'getVerbs') + + await expect(brain.related({ from: a })).rejects.toBeInstanceOf(GraphIndexNotReadyError) + await expect(brain.related({ from: a })).rejects.toThrow(/adjacency-residency/) + + expect(getNounSpy).not.toHaveBeenCalled() + expect(getVerbsSpy).not.toHaveBeenCalled() + + delete internals.graphIndex.healthReport + }) +}) + +describe('health gate (b) — unledgered is unknown: never blocks a serving provider', () => { + it('serving:true with an unledgered family and no failing invariant serves normally; at most one narration', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + internals.metadataIndex.healthReport = () => + healthReport({ + provider: 'metadata', + serving: true, + healthy: true, + invariants: [], + unledgered: ['canonical-verb-coverage'] + }) + + const warnSpy = vi.spyOn(prodLog, 'warn') + + const r1 = await brain.find({ where: { team: 'atlas' } }) + const r2 = await brain.find({ where: { team: 'atlas' } }) + expect(r1.length).toBe(1) + expect(r2.length).toBe(1) + + const narrations = warnSpy.mock.calls.filter( + ([msg]) => typeof msg === 'string' && msg.includes('canonical-verb-coverage') + ) + expect(narrations.length).toBe(1) // one narration at most across both reads (same generation) + + delete internals.metadataIndex.healthReport + }) +}) + +describe('health gate (c) — degraded-but-serving narrates once per generation', () => { + it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + let generation = 1 + internals.index.healthReport = () => + healthReport({ + provider: 'vector', + serving: true, + healthy: false, + invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail: 'counter drift' })], + generation + }) + + const warnSpy = vi.spyOn(prodLog, 'warn') + const countNarrations = () => + warnSpy.mock.calls.filter(([msg]) => typeof msg === 'string' && msg.includes('stale-vector-counter')).length + + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + expect(countNarrations()).toBe(1) // same generation both times — one narration + + generation = 2 + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + expect(countNarrations()).toBe(2) // generation bumped — a second narration + + delete internals.index.healthReport + }) +}) + +describe('health gate (d) — interim isReady()-only path (no healthReport) is unchanged', () => { + it('isReady() === true serves; isReady() === false refuses via the typed NotReady error', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + internals.metadataIndex.isReady = () => true + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + + internals.metadataIndex.isReady = () => false + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError) + + delete internals.metadataIndex.isReady + }) +}) + +describe('health gate (e) — open builds; the first read never does', () => { + it('disableAutoRebuild:true on a populated store: open narrates + builds; the first find() triggers zero rebuilds', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-healthgate-open-')) + dirs.push(dir) + + const writer = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false, + silent: true, + disableAutoRebuild: true + }) + await writer.init() + brains.push(writer) + await writer.add({ data: 'row one', type: NounType.Document, metadata: { team: 'atlas' } }) + await writer.flush() + await brains.pop()!.close() + + const warnSpy = vi.spyOn(prodLog, 'warn') + const reader = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false, + silent: true, + disableAutoRebuild: true + }) + const internals = internalsOf(reader) + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded') + + await reader.init() + brains.push(reader) + + expect(rebuildSpy).toHaveBeenCalledTimes(1) // open() built it, exactly once + expect( + warnSpy.mock.calls.some( + ([msg]) => typeof msg === 'string' && msg.includes('open() is building') + ) + ).toBe(true) + + rebuildSpy.mockClear() + const rows = await reader.find({ where: { team: 'atlas' } }) + expect(rebuildSpy).toHaveBeenCalledTimes(0) // the read never builds + expect(rows.length).toBe(1) + }, 30000) +}) + +describe('health gate (f) — the ceremony door: explicit rebuild bypasses invariant consultation', () => { + it("repairIndex({ rebuild: ['graph'] }) rebuilds unconditionally without consulting validateInvariants", async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'x', type: NounType.Concept }) + await brain.flush() + + const internals = internalsOf(brain) + let validateCalls = 0 + internals.graphIndex.validateInvariants = async () => { + validateCalls++ + return healthReport({ provider: 'graph' }) + } + const rebuildSpy = vi.spyOn(internals.graphIndex, 'rebuild') + + const report = await brain.repairIndex({ rebuild: ['graph'] }) + + expect(rebuildSpy).toHaveBeenCalledTimes(1) + expect(validateCalls).toBe(0) // the door never consults validateInvariants to decide + + const graphFamily = report.families.find((f) => f.family === 'provider:graph') + expect(graphFamily?.rebuilt).toBe(true) + expect(graphFamily?.checked).toBe(true) + expect(graphFamily?.reason).toBe('explicit rebuild requested') + + delete internals.graphIndex.validateInvariants + }) + + it('bare repairIndex() on a healthy provider calls no rebuild()', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'x', type: NounType.Concept }) + await brain.flush() + + const internals = internalsOf(brain) + internals.graphIndex.validateInvariants = async () => healthReport({ provider: 'graph', healthy: true, serving: true }) + const rebuildSpy = vi.spyOn(internals.graphIndex, 'rebuild') + + await brain.repairIndex() + + expect(rebuildSpy).not.toHaveBeenCalled() + + delete internals.graphIndex.validateInvariants + }) +}) + +describe('health gate (g) — a throwing healthReport() is a contract violation, never read as healthy', () => { + it('healthReport() that throws refuses loudly with the typed NotReady error naming the throw', async () => { + const brain = new Brainy(createTestConfig({ silent: true })) + await brain.init() + brains.push(brain) + await brain.add({ data: 'row', type: NounType.Document, metadata: { team: 'atlas' } }) + await brain.flush() + + const internals = internalsOf(brain) + internals.index.healthReport = () => { + throw new Error('accelerator: mmap window busy') + } + + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(VectorIndexNotReadyError) + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/mmap window busy/) + + delete internals.index.healthReport + }) +}) diff --git a/tests/lifecycle/README.md b/tests/lifecycle/README.md new file mode 100644 index 00000000..e6456f19 --- /dev/null +++ b/tests/lifecycle/README.md @@ -0,0 +1,16 @@ +# The Lifecycle Lane + +One brain, driven through founding, a working day, a clean restart, a +crash, a repair, and a second life, checked chapter by chapter against an +independent shadow-model referee (`biographyHarness.ts`). It catches +COMPOSITION regressions unit tests miss — a store fine in one process but +broken across a restart/crash/repair. Runs on the plain JS engine, so it +gates every commit. + +Run it: `npx vitest run tests/lifecycle --pool=forks` + +A red names the chapter label, the id, and expected-vs-actual — diagnosable +from the message alone. `biography.test.ts` is split into two `it` blocks +(Ch1-3, then Ch4-6) purely for reporting; it is still ONE fixed-order story. +Chapters must never be reordered, skipped, or made conditional, and a +failing chapter's assertion must never be weakened to force green. diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts new file mode 100644 index 00000000..305d7a99 --- /dev/null +++ b/tests/lifecycle/biography.test.ts @@ -0,0 +1,429 @@ +/** + * @module tests/lifecycle/biography + * @description THE LIFECYCLE LANE — see `tests/lifecycle/README.md` for what + * this proves and how to run it. One scenario, "the working store": a single + * brain driven through founding, a working day, a clean restart, a crash, a + * repair, and a second life, verified chapter by chapter against an + * independent shadow-model referee (`biographyHarness.ts`). + * + * Split into two `it` blocks so a currently-failing later chapter (see the + * second block's header comment — a live engine finding, not a defect in + * this lane) never hides the earlier chapters' passing coverage. The two + * blocks share one brain's directory and one shadow model, run in the SAME + * fixed order the single scenario always has (`describe.sequential` below + * exists to say so explicitly, though vitest's own default is sequential + * within a file) — this is a split for REPORTING clarity, not a reordering + * or conditional skip of any chapter. + */ +import { describe, it, expect } from 'vitest' +import * as fs from 'node:fs' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import type { Brainy } from '../../src/brainy.js' +import type { AddParams, RelateParams, UpdateParams, UpdateRelationParams } from '../../src/index.js' +import { abandonAsCrashed, makeTempDir, openBrain, uid } from '../helpers/durabilityKillMatrix.js' +import { + createModel, + getCanonicalCountsFor, + modelAdd, + modelDelete, + modelRelate, + modelUpdate, + modelUpdateRelation, + recordVfsFileWrite, + snapshotVfsBaseline, + verifyChapter, + type HubCheck, + type ShadowModel +} from './biographyHarness.js' + +const STATUSES = ['active', 'pending', 'closed', 'archived'] as const + +/** Cycle a status value to the next one in the fixed rotation — used so + * Ch2's 40 updates provably MOVE entities across find() buckets rather than + * risking a no-op reassignment of the same value. */ +function nextStatus(current: unknown): (typeof STATUSES)[number] { + const currentStr = typeof current === 'string' ? current : STATUSES[0] + const idx = STATUSES.indexOf(currentStr as (typeof STATUSES)[number]) + return STATUSES[(idx < 0 ? 0 : idx + 1) % STATUSES.length] +} + +// --------------------------------------------------------------------------- +// Shared biography state — set up by the first `it`, consumed by the second. +// The two blocks are one continuous story told in two named pieces; nothing +// here resets or diverges between them. +// --------------------------------------------------------------------------- +let dir: string +let model: ShadowModel +let brain: Brainy +let hubs: HubCheck[] +let employees: string[] +let customers: string[] +let invoices: string[] +let tasks: string[] +let projects: string[] +let nonHub: string[] + +// ---- Wrappers: every call to the real brain updates the shadow model in +// the same statement, so the two can never drift apart by construction. +// Defined once, closing over the `let` bindings above so both `it` blocks +// (and any future reopen inside them) operate on the current brain/model. +async function doAdd(label: string, params: Omit): Promise { + const id = uid(label) + await brain.add({ ...params, id }) + modelAdd(model, id, { + type: params.type, + subtype: params.subtype, + metadata: params.metadata ?? {}, + visibility: params.visibility + }) + return id +} + +async function doUpdate(id: string, patch: Omit): Promise { + await brain.update({ ...patch, id }) + modelUpdate(model, id, { metadata: patch.metadata, merge: patch.merge, visibility: patch.visibility }) +} + +async function doRemove(id: string): Promise { + await brain.remove(id) + modelDelete(model, id) +} + +async function doRelate(params: RelateParams): Promise { + const id = await brain.relate(params) + modelRelate(model, id, { + from: params.from, + to: params.to, + type: params.type, + subtype: params.subtype, + metadata: params.metadata + }) + return id +} + +async function doUpdateRelation(id: string, patch: Omit): Promise { + await brain.updateRelation({ ...patch, id }) + modelUpdateRelation(model, id, { metadata: patch.metadata, merge: patch.merge }) +} + +async function doVfsWrite(path: string, content: string): Promise { + await brain.vfs.writeFile(path, content) + recordVfsFileWrite(model) +} + +describe.sequential('lifecycle — the working store', () => { + it( + 'Ch1 FOUNDING -> Ch2 A WORKING DAY -> Ch3 CLEAN RESTART: every read serves truth', + async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = makeTempDir() + model = createModel() + + // logAuthority: 'adopt' from the first open, mirrored across every + // reopen — see write-flow-production-shape.test.ts, which the later + // crash chapter's at-ack law is pinned against. + brain = await openBrain(dir, { logAuthority: 'adopt' }) + + // ================================================================= + // CHAPTER 1 — FOUNDING + // ================================================================= + // Baseline MUST be snapshotted before any biography act — it is the + // VFS root's own system-tier footprint, measured, never hardcoded. + await snapshotVfsBaseline(brain, model) + + employees = [] + for (let i = 0; i < 20; i++) { + employees.push( + await doAdd(`emp-${i}`, { + data: `employee record ${i}`, + type: NounType.Person, + subtype: 'employee', + metadata: { status: STATUSES[i % STATUSES.length], department: ['engineering', 'sales', 'support'][i % 3] } + }) + ) + } + customers = [] + for (let i = 0; i < 20; i++) { + customers.push( + await doAdd(`cust-${i}`, { + data: `customer record ${i}`, + type: NounType.Person, + subtype: 'customer', + metadata: { status: STATUSES[i % STATUSES.length], tier: i % 2 === 0 ? 'gold' : 'standard' } + }) + ) + } + invoices = [] + for (let i = 0; i < 30; i++) { + invoices.push( + await doAdd(`inv-${i}`, { + data: `invoice record ${i}`, + type: NounType.Document, + subtype: 'invoice', + metadata: { status: STATUSES[i % STATUSES.length], amount: 100 + i * 17 } + }) + ) + } + tasks = [] + for (let i = 0; i < 25; i++) { + tasks.push( + await doAdd(`task-${i}`, { + data: `task record ${i}`, + type: NounType.Task, + subtype: 'milestone', + metadata: { status: STATUSES[i % STATUSES.length], priority: (i % 5) + 1 } + }) + ) + } + projects = [] + for (let i = 0; i < 25; i++) { + projects.push( + await doAdd(`proj-${i}`, { + data: `project record ${i}`, + type: NounType.Project, + metadata: { status: STATUSES[i % STATUSES.length], budget: 1000 * (i + 1) } + }) + ) + } + expect(employees.length + customers.length + invoices.length + tasks.length + projects.length).toBe(120) + + // Five hubs (proj-0..proj-4) fan out to tasks (Contains) and employees + // (WorksWith); a residual band of invoice->customer RelatedTo edges is + // unrelated to any hub. Hubs are never touched again for the rest of + // the biography, so they stay valid adjacency samples in every chapter. + const hubIds = projects.slice(0, 5) + for (let h = 0; h < 5; h++) { + for (let k = 0; k < 15; k++) { + const taskIdx = (h * 5 + k) % tasks.length + await doRelate({ from: hubIds[h], to: tasks[taskIdx], type: VerbType.Contains, subtype: 'delivers' }) + } + for (let k = 0; k < 10; k++) { + const empIdx = (h * 4 + k) % employees.length + await doRelate({ from: hubIds[h], to: employees[empIdx], type: VerbType.WorksWith }) + } + } + for (let j = 0; j < 25; j++) { + await doRelate({ from: invoices[j], to: customers[j % customers.length], type: VerbType.RelatedTo, subtype: 'billed-to' }) + } + expect(model.relations.size).toBe(150) + + // A handful of VFS files. + for (let i = 0; i < 5; i++) { + await doVfsWrite(`/report-${i}.txt`, `founding report ${i}`) + } + + await brain.flush() + + hubs = hubIds.map((id) => ({ id, typeFilters: [VerbType.Contains, VerbType.WorksWith] })) + await verifyChapter(brain, model, 'Ch1 FOUNDING', { hubs, bucketField: 'status' }) + + // ================================================================= + // CHAPTER 2 — A WORKING DAY + // ================================================================= + // Non-hub pool for every mutation below. + nonHub = [...employees, ...customers, ...invoices, ...tasks, ...projects.slice(5)] + + // 40 updates that provably MOVE entities across find() status buckets. + const updateTargets = nonHub.slice(0, 40) + for (const id of updateTargets) { + const current = model.entities.get(id)!.metadata.status + await doUpdate(id, { metadata: { status: nextStatus(current) } }) + } + + // 10 visibility flips (public -> internal). + const visibilityTargets = nonHub.slice(40, 50) + for (const id of visibilityTargets) { + await doUpdate(id, { visibility: 'internal' }) + } + + // 15 deletes — some hub members (their edges cascade away), 3 of them + // earmarked for Ch6's resurrection. + const resurrectIds = [tasks[0], tasks[1], employees[0]] + const otherDeletes = [ + tasks[2], tasks[3], tasks[4], tasks[5], tasks[6], + employees[1], employees[2], employees[3], + customers[0], customers[1], customers[2], customers[3] + ] + const ch2DeleteTargets = [...resurrectIds, ...otherDeletes] + expect(ch2DeleteTargets.length).toBe(15) + for (const id of ch2DeleteTargets) { + await doRemove(id) + } + + // 20 new adds. + const ch2NewTypes = [NounType.Person, NounType.Document, NounType.Task] + for (let i = 0; i < 20; i++) { + await doAdd(`ch2-new-${i}`, { + data: `working-day addition ${i}`, + type: ch2NewTypes[i % ch2NewTypes.length], + subtype: 'ad-hoc', + metadata: { status: STATUSES[i % STATUSES.length] } + }) + } + + // 10 updateRelation metadata patches — read AFTER the deletes above, + // so only relations the cascade left alive are ever targeted. + const survivingRelationIds = [...model.relations.keys()].slice(0, 10) + expect(survivingRelationIds.length).toBe(10) + for (const relId of survivingRelationIds) { + await doUpdateRelation(relId, { metadata: { reviewed: true } }) + } + + await brain.flush() + await verifyChapter(brain, model, 'Ch2 A WORKING DAY', { hubs, bucketField: 'status' }) + + // ================================================================= + // CHAPTER 3 — CLEAN RESTART + // ================================================================= + await brain.close() + brain = await openBrain(dir, { logAuthority: 'adopt' }) + await verifyChapter(brain, model, 'Ch3 CLEAN RESTART', { hubs, bucketField: 'status' }) + + // Leave the brain closed and the directory intact for the next `it` + // (the biography continues there) — do NOT remove `dir` here. + await brain.close() + }, + 300000 + ) + + /** + * Ch4 CRASH is a LIVE ENGINE FINDING, not a defect in this lane (see + * README.md and the project report this lane's build produced): after a + * crash (writes acked at commit but never flushed, the process abandoned + * exactly as `abandonAsCrashed` models, then reopened), canonical storage + * (`get()`), the vector index, and `getNounCount()`/`getCanonicalCounts()` + * all correctly recover every acked write — but the METADATA INDEX behind + * `find({ where })` recovers NONE of the crash-window's acked writes + * (neither new adds nor metadata updates to pre-existing entities), even + * though `getIndexStatus()` reports `projections.metadata.synchronous: + * true`. `repairIndex()` cannot close the gap either: its own report names + * `provider:metadata` as `checked: false, skipped: "no + * validateInvariants/rebuild contract"`. The assertion below states the + * TRUE contract (find() must agree with get()) and is expected to fail + * against the current engine — it must never be loosened to paper over + * this. Ch5/Ch6 are written in full below it and will start running the + * moment this gap is closed; they are not dead code, they are blocked code. + */ + // RELEASE-BLOCKING FINDING (the kill-matrix convention: assert the CONTRACT, + // mark `.fails`, never weaken): after a crash + adopt reopen, the JS metadata + // index computes its watermark verdict but nothing consumes 'catchup' + // (metadataIndex.ts loadWatermarkVerdict) — find() serves the pre-crash + // index while get()/counts recover. The catchup wiring is the cure; when it + // lands this `.fails` marker MUST be removed (vitest will force it: a + // passing `.fails` test is itself a failure). + it.fails( + 'Ch4 CRASH -> Ch5 REPAIR -> Ch6 SECOND LIFE: continues the Ch3 store', + async () => { + try { + brain = await openBrain(dir, { logAuthority: 'adopt' }) + + // =============================================================== + // CHAPTER 4 — CRASH + // =============================================================== + const ch4Types = [NounType.Person, NounType.Document, NounType.Task, NounType.Project] + for (let i = 0; i < 10; i++) { + await doAdd(`ch4-new-${i}`, { + data: `crash-window addition ${i}`, + type: ch4Types[i % ch4Types.length], + metadata: { status: STATUSES[i % STATUSES.length] } + }) + } + const ch4UpdateTargets = nonHub.slice(50, 55) // invoices[10..14] — untouched so far + for (const id of ch4UpdateTargets) { + await doUpdate(id, { metadata: { status: 'active' } }) + } + // NO flush — abandon exactly the way process death would (the + // at-ack law: every write already awaited above must survive). + await abandonAsCrashed(brain) + brain = await openBrain(dir, { logAuthority: 'adopt' }) + await verifyChapter(brain, model, 'Ch4 CRASH', { hubs, bucketField: 'status' }) + + // =============================================================== + // CHAPTER 5 — REPAIR + // =============================================================== + const report = await brain.repairIndex() + for (const family of report.families) { + const accounted = + family.checked === true || (family.checked === false && typeof family.skipped === 'string' && family.skipped.length > 0) + expect( + accounted, + `[Ch5 REPAIR] family '${family.family}' must be checked or explicitly skipped with a reason; got ${JSON.stringify(family)}` + ).toBe(true) + } + // A healthy store: repair must change nothing the model doesn't + // already expect — verifyChapter against the UNCHANGED model proves it. + await verifyChapter(brain, model, 'Ch5 REPAIR', { hubs, bucketField: 'status' }) + + // =============================================================== + // CHAPTER 6 — SECOND LIFE + // =============================================================== + const ch6Types = [NounType.Person, NounType.Document, NounType.Task, NounType.Project] + for (let i = 0; i < 10; i++) { + await doAdd(`ch6-new-${i}`, { + data: `second-life addition ${i}`, + type: ch6Types[i % ch6Types.length], + metadata: { status: STATUSES[i % STATUSES.length] } + }) + } + const ch6UpdateTargets = nonHub.slice(55, 65) // invoices[15..24] — untouched so far + expect(ch6UpdateTargets.every((id) => model.entities.get(id)!.alive)).toBe(true) + for (const id of ch6UpdateTargets) { + await doUpdate(id, { metadata: { status: 'closed' } }) + } + const ch6DeleteTargets = nonHub + .slice(65, 90) // invoices[25..29] + tasks[0..19] (some already dead — filtered below) + .filter((id) => model.entities.get(id)!.alive) + .slice(0, 7) + expect(ch6DeleteTargets.length).toBe(7) + for (const id of ch6DeleteTargets) { + await doRemove(id) + } + + // Resurrection: the SAME three ids Ch2 deleted, reinserted with + // BRAND-NEW metadata — the model expects the new metadata only. + await doAdd('task-0', { data: 'resurrected task 0', type: NounType.Task, subtype: 'milestone', metadata: { status: 'active', resurrected: true } }) + await doAdd('task-1', { data: 'resurrected task 1', type: NounType.Task, subtype: 'milestone', metadata: { status: 'pending', resurrected: true } }) + await doAdd('emp-0', { data: 'resurrected employee 0', type: NounType.Person, subtype: 'employee', metadata: { status: 'active', resurrected: true } }) + expect(tasks[0]).toBe(uid('task-0')) // same id as Ch1/Ch2 — the resurrection-adjacent shape + + await brain.close() + brain = await openBrain(dir, { logAuthority: 'adopt' }) + await verifyChapter(brain, model, 'Ch6 SECOND LIFE', { hubs, bucketField: 'status' }) + + // Final, standalone getCanonicalCounts() exactness check (beyond + // verifyChapter's own (f) leg) — the whole ledger, in one shot. + const finalCounts = await getCanonicalCountsFor(brain) + const aliveEntities = [...model.entities.values()].filter((e) => e.alive) + const alivePublicEntities = aliveEntities.filter((e) => (e.visibility ?? 'public') === 'public') + const aliveVerbs = model.relations.size + expect(finalCounts, 'final getCanonicalCounts() exactness — Ch6 SECOND LIFE').toEqual({ + nouns: { + counted: alivePublicEntities.length + model.vfsFileNouns, + all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns + }, + verbs: { + counted: aliveVerbs + model.vfsContainsVerbs, + all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs + }, + suspect: false + }) + } finally { + await brain.close().catch(() => {}) + // Best-effort, retried: a still-draining background persistence + // write (e.g. count/index write-through) can race a single rmSync + // and leave a partial directory behind — retry a couple of times + // rather than let this temp dir leak. + for (let attempt = 0; attempt < 3; attempt++) { + try { + fs.rmSync(dir, { recursive: true, force: true }) + if (!fs.existsSync(dir)) break + } catch { + // ignore and retry + } + await new Promise((resolve) => setTimeout(resolve, 100)) + } + } + }, + 300000 + ) +}) diff --git a/tests/lifecycle/biographyHarness.ts b/tests/lifecycle/biographyHarness.ts new file mode 100644 index 00000000..ca15b36a --- /dev/null +++ b/tests/lifecycle/biographyHarness.ts @@ -0,0 +1,389 @@ +/** + * @module tests/lifecycle/biographyHarness + * @description The referee for the LIFECYCLE LANE (see `biography.test.ts`): + * a plain in-memory SHADOW MODEL of a brain's contents, updated by every act + * the biography performs (add/update/remove/relate/updateRelation/vfs writes), + * plus `verifyChapter()`, which asserts the live brain agrees with the model + * after every chapter. No engine code runs inside the model — it is an + * independent ledger, not a mirror of the implementation under test. + * + * COUNT SEMANTICS this harness encodes (verified against the live engine, + * not assumed — see the module-level comments below for how each was + * confirmed): + * + * - `getNounCount()` / `getVerbCount()` count PUBLIC-tier alive records only + * (visibility absent or `'public'`) — `'internal'` and `'system'` are both + * excluded. `storage.getCanonicalCounts()` mirrors that same PUBLIC-only + * scalar as `counted`, and additionally reports `all` — every tier, + * unfiltered — as the coverage-ledger denominator (see + * tests/integration/canonical-count-ledger.test.ts). + * - `brain.vfs.writeFile()` for a brand-new file at a path directly under the + * VFS root creates exactly ONE new File noun plus ONE new `Contains` verb + * (root -> file), and BOTH are ordinary PUBLIC records (no visibility + * field is set) — so they count toward `getNounCount()`/`getVerbCount()` + * as well as the canonical `all` scalars. Only the VFS ROOT entity itself + * is `'system'`-tier (created once, at `init()`, before any biography + * chapter runs) — that lone record is the only hidden-tier footprint the + * model does not construct explicitly, so it is captured empirically via + * `snapshotVfsBaseline()` immediately after `init()` rather than hardcoded. + * - `related()` filters edges by the RELATION's own visibility tier, not by + * the visibility of the entities the edge connects — flipping an entity to + * `'internal'` does not hide its edges from `related()`. This lane never + * sets relation visibility, so every relation the model tracks is exactly + * as reachable as its presence in `model.relations` implies. + * - `remove()` cascades: every relation touching the removed entity (as + * `from` or `to`) is hard-deleted along with it. The model mirrors this by + * deleting the relation entirely from `model.relations` (no relation + * "alive" flag — presence in the map IS aliveness). + */ +import { expect } from 'vitest' +import type { Brainy } from '../../src/brainy.js' +import type { NounType, VerbType } from '../../src/types/graphTypes.js' +import type { EntityVisibility, StorageAdapter } from '../../src/coreTypes.js' + +/** + * One entity's complete lifecycle-relevant state, as the biography's acts + * leave it. `alive: false` means the model believes the id has been removed + * — the entry is KEPT (never deleted from the map) so `verifyChapter` can + * assert the negative half of the contract: a dead id must read as `null`. + */ +export interface ShadowEntity { + type: NounType + subtype?: string + metadata: Record + visibility?: EntityVisibility + alive: boolean +} + +/** + * One relation's complete lifecycle-relevant state. There is no `alive` + * flag here — presence in {@link ShadowModel.relations} IS aliveness, + * mirroring the engine's hard delete of the canonical verb record on + * cascade (see the module header). + */ +export interface ShadowRelation { + from: string + to: string + type: VerbType + subtype?: string + metadata: Record +} + +/** + * The independent truth ledger the biography updates on every act it + * performs. `verifyChapter` checks the live brain against this — never the + * other way around. + */ +export interface ShadowModel { + entities: Map + relations: Map + /** + * `getCanonicalCounts()` nouns.all / verbs.all captured right after + * `init()`, before chapter 1 — the VFS root's own system-tier footprint. + * Set once via {@link snapshotVfsBaseline}; never hardcoded. + */ + vfsBaselineNouns: number + vfsBaselineVerbs: number + /** + * Public nouns/verbs created by `vfs.writeFile()` for a brand-new file at + * a flat top-level path: exactly one File noun + one Contains verb per + * call (see the module header). Bumped by {@link recordVfsFileWrite}. + */ + vfsFileNouns: number + vfsContainsVerbs: number +} + +/** A fresh, empty shadow model — call once before chapter 1. */ +export function createModel(): ShadowModel { + return { + entities: new Map(), + relations: new Map(), + vfsBaselineNouns: 0, + vfsBaselineVerbs: 0, + vfsFileNouns: 0, + vfsContainsVerbs: 0 + } +} + +/** Narrow, documented private-storage access (the same style already used by + * `tests/helpers/durabilityKillMatrix.ts`'s `storeOf()`), needed because + * `getCanonicalCounts()` lives on the storage adapter, not on `Brainy`. */ +function storageOf(brain: Brainy): StorageAdapter { + return (brain as unknown as { storage: StorageAdapter }).storage +} + +/** Public wrapper around the private-storage `getCanonicalCounts()` read, so + * callers never need their own private-access cast — used internally by + * {@link snapshotVfsBaseline} and {@link verifyChapter}, and by + * `biography.test.ts` for its final standalone exactness check. */ +export async function getCanonicalCountsFor(brain: Brainy): ReturnType> { + const storage = storageOf(brain) + if (!storage.getCanonicalCounts) { + throw new Error( + 'lifecycle lane: the storage adapter under test has no getCanonicalCounts() — the canonical-count-exactness leg of this lane is unrepresentable without it.' + ) + } + return storage.getCanonicalCounts() +} + +/** + * Snapshot the VFS root's own hidden-tier footprint. Call exactly once, + * immediately after `init()` and before chapter 1 does anything — this is + * the ONE baseline offset the model does not construct by hand (see the + * module header for why: the root is `'system'`-tier plumbing the biography + * never explicitly creates). + */ +export async function snapshotVfsBaseline(brain: Brainy, model: ShadowModel): Promise { + const counts = await getCanonicalCountsFor(brain) + model.vfsBaselineNouns = counts.nouns.all + model.vfsBaselineVerbs = counts.verbs.all +} + +/** + * Record one `brain.vfs.writeFile()` call for a brand-new file at a flat + * top-level path (no intermediate directories). Bumps both the noun and verb + * VFS counters by one, matching the engine's actual write path exactly (see + * the module header) — never call this for an overwrite of an existing path, + * a nested path (which would also vivify intermediate directory nouns/edges, + * a different, unmodeled shape), or the biography loses its exactness. + */ +export function recordVfsFileWrite(model: ShadowModel): void { + model.vfsFileNouns += 1 + model.vfsContainsVerbs += 1 +} + +/** Record a fresh `add()` (or a Ch6 resurrection — `Map.set` fully replaces + * whatever a prior dead entry held, which is exactly the "new metadata only" + * contract a resurrection must honor). */ +export function modelAdd( + model: ShadowModel, + id: string, + entity: { type: NounType; subtype?: string; metadata: Record; visibility?: EntityVisibility } +): void { + model.entities.set(id, { + type: entity.type, + subtype: entity.subtype, + metadata: { ...entity.metadata }, + visibility: entity.visibility, + alive: true + }) +} + +/** Record an `update()` — merges metadata by default, matching the engine's + * `merge: true` default; pass `merge: false` to mirror a full replace. */ +export function modelUpdate( + model: ShadowModel, + id: string, + patch: { metadata?: Record; merge?: boolean; visibility?: EntityVisibility } +): void { + const existing = model.entities.get(id) + if (!existing || !existing.alive) { + throw new Error(`shadow model: update() targeted ${id}, which the model does not have alive — biography sequencing bug`) + } + if (patch.metadata) { + existing.metadata = patch.merge === false ? { ...patch.metadata } : { ...existing.metadata, ...patch.metadata } + } + if (patch.visibility !== undefined) { + existing.visibility = patch.visibility + } +} + +/** Record a `remove()` — marks the entity dead (entry retained, per + * {@link ShadowEntity}) and cascades: every relation touching it, in either + * direction, is hard-deleted from the model too (matching the engine). */ +export function modelDelete(model: ShadowModel, id: string): void { + const existing = model.entities.get(id) + if (!existing || !existing.alive) { + throw new Error(`shadow model: remove() targeted ${id}, which the model does not have alive — biography sequencing bug`) + } + existing.alive = false + for (const [relId, rel] of model.relations) { + if (rel.from === id || rel.to === id) model.relations.delete(relId) + } +} + +/** Record a `relate()` — `id` is the relation id the real call returned. */ +export function modelRelate( + model: ShadowModel, + id: string, + relation: { from: string; to: string; type: VerbType; subtype?: string; metadata?: Record } +): void { + model.relations.set(id, { + from: relation.from, + to: relation.to, + type: relation.type, + subtype: relation.subtype, + metadata: { ...(relation.metadata ?? {}) } + }) +} + +/** Record an `updateRelation()` metadata patch — merges by default. */ +export function modelUpdateRelation( + model: ShadowModel, + id: string, + patch: { metadata?: Record; merge?: boolean } +): void { + const existing = model.relations.get(id) + if (!existing) { + throw new Error(`shadow model: updateRelation() targeted ${id}, which the model does not have — biography sequencing bug`) + } + if (patch.metadata) { + existing.metadata = patch.merge === false ? { ...patch.metadata } : { ...existing.metadata, ...patch.metadata } + } +} + +/** Order-independent structural equality for plain JSON-shaped metadata. */ +function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true + if (typeof a !== typeof b) return false + if (a === null || b === null) return a === b + if (typeof a !== 'object') return false + const aKeys = Object.keys(a as Record) + const bKeys = Object.keys(b as Record) + if (aKeys.length !== bKeys.length) return false + for (const k of aKeys) { + if (!deepEqual((a as Record)[k], (b as Record)[k])) return false + } + return true +} + +/** One hub entity to sample for the `related()` adjacency check, plus the + * verb type(s) it is known (by biography construction) to have OUT-edges + * of, so the type-filtered variant is exercised too. */ +export interface HubCheck { + id: string + typeFilters: VerbType[] +} + +/** Options steering one `verifyChapter()` call. */ +export interface VerifyOptions { + /** Hub entities to sample for the `related()` adjacency check. */ + hubs: HubCheck[] + /** The metadata field `find()` bucket-checks against (a bare string field + * every alive entity may or may not carry — distinct values present among + * ALIVE model entities are discovered automatically each call, so a + * chapter that moves entities across buckets is re-checked exactly). */ + bucketField: string +} + +/** + * Assert the live brain agrees with the model, in full, after one chapter. + * Every failure message names the chapter `label`, the id (where + * applicable), and expected-vs-actual — a red here must be diagnosable from + * the assertion message alone, with no need to re-read this file. + */ +export async function verifyChapter(brain: Brainy, model: ShadowModel, label: string, opts: VerifyOptions): Promise { + // (a) + (b): every alive entity reads back exactly as modeled; every dead + // entity reads as null. + for (const [id, entity] of model.entities) { + const live = await brain.get(id) + if (entity.alive) { + expect(live, `[${label}] alive entity ${id} (type=${entity.type}) must be readable via get(), got null`).not.toBeNull() + const e = live! + expect(e.type, `[${label}] entity ${id} .type mismatch: expected ${entity.type}, got ${e.type}`).toBe(entity.type) + expect(e.subtype, `[${label}] entity ${id} .subtype mismatch: expected ${JSON.stringify(entity.subtype)}, got ${JSON.stringify(e.subtype)}`).toBe(entity.subtype) + expect( + e.visibility, + `[${label}] entity ${id} .visibility mismatch: expected ${JSON.stringify(entity.visibility)}, got ${JSON.stringify(e.visibility)}` + ).toBe(entity.visibility) + const metaMatches = deepEqual(e.metadata ?? {}, entity.metadata) + expect( + metaMatches, + `[${label}] entity ${id} .metadata mismatch: expected ${JSON.stringify(entity.metadata)}, got ${JSON.stringify(e.metadata)}` + ).toBe(true) + } else { + expect(live, `[${label}] dead entity ${id} (type=${entity.type}) must read as null, got ${JSON.stringify(live)}`).toBeNull() + } + } + + // (c) find({ where: { : value } }) returns exactly the + // model's matching alive set, per distinct value currently present. + const bucketValues = new Set() + for (const entity of model.entities.values()) { + if (!entity.alive) continue + const v = entity.metadata[opts.bucketField] + if (typeof v === 'string') bucketValues.add(v) + } + for (const value of bucketValues) { + const expectedIds = [...model.entities.entries()] + .filter(([, e]) => e.alive && e.metadata[opts.bucketField] === value) + .map(([id]) => id) + .sort() + const results = await brain.find({ + where: { [opts.bucketField]: value } as Record, + includeInternal: true, + limit: 100000 + }) + const actualIds = results.map((r) => r.id).sort() + expect( + actualIds, + `[${label}] find({ where: { ${opts.bucketField}: ${JSON.stringify(value)} } }) mismatch: expected ${expectedIds.length} ids ${JSON.stringify(expectedIds)}, got ${actualIds.length} ids ${JSON.stringify(actualIds)}` + ).toEqual(expectedIds) + } + + // (d) related(id) / related(id, { type }) for the hub sample matches the + // model's adjacency exactly (out-edges — related(id) is shorthand for + // { from: id }). + for (const hub of opts.hubs) { + const expectedAll = [...model.relations.entries()] + .filter(([, r]) => r.from === hub.id) + .map(([id]) => id) + .sort() + const liveAll = await brain.related({ from: hub.id, limit: 100000 }) + const actualAllIds = liveAll.map((r) => r.id).sort() + expect( + actualAllIds, + `[${label}] related(${hub.id}) mismatch: expected ${expectedAll.length} ids ${JSON.stringify(expectedAll)}, got ${actualAllIds.length} ids ${JSON.stringify(actualAllIds)}` + ).toEqual(expectedAll) + + for (const typeFilter of hub.typeFilters) { + const expectedTyped = [...model.relations.entries()] + .filter(([, r]) => r.from === hub.id && r.type === typeFilter) + .map(([id]) => id) + .sort() + const liveTyped = await brain.related({ from: hub.id, type: typeFilter, limit: 100000 }) + const actualTypedIds = liveTyped.map((r) => r.id).sort() + expect( + actualTypedIds, + `[${label}] related(${hub.id}, { type: '${typeFilter}' }) mismatch: expected ${expectedTyped.length} ids ${JSON.stringify(expectedTyped)}, got ${actualTypedIds.length} ids ${JSON.stringify(actualTypedIds)}` + ).toEqual(expectedTyped) + } + } + + // (e) getNounCount() / getVerbCount(): PUBLIC-tier alive records + // (visibility absent/'public'; 'internal' and 'system' both excluded — see + // the module header) plus the VFS's own public contributions. + const alivePublicNouns = [...model.entities.values()].filter((e) => e.alive && (e.visibility ?? 'public') === 'public').length + const aliveVerbs = model.relations.size + const expectedNounCount = alivePublicNouns + model.vfsFileNouns + const expectedVerbCount = aliveVerbs + model.vfsContainsVerbs + expect( + await brain.getNounCount(), + `[${label}] getNounCount() mismatch: expected ${expectedNounCount} (alive public entities ${alivePublicNouns} + vfs file nouns ${model.vfsFileNouns})` + ).toBe(expectedNounCount) + expect( + await brain.getVerbCount(), + `[${label}] getVerbCount() mismatch: expected ${expectedVerbCount} (alive relations ${aliveVerbs} + vfs contains verbs ${model.vfsContainsVerbs})` + ).toBe(expectedVerbCount) + + // (f) getCanonicalCounts(): ALL-visibility scalars (every tier) equal the + // model's alive totals including hidden tiers, plus the VFS's own + // contributions (both file nouns/verbs AND the once-measured root + // baseline). suspect must be false — every delete in this biography goes + // through brain.remove(), which always proves the record it decrements. + const ledger = await getCanonicalCountsFor(brain) + const aliveAllNouns = [...model.entities.values()].filter((e) => e.alive).length + const expectedNounsAll = aliveAllNouns + model.vfsFileNouns + model.vfsBaselineNouns + const expectedVerbsAll = aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs + expect( + ledger.nouns.all, + `[${label}] getCanonicalCounts().nouns.all mismatch: expected ${expectedNounsAll} (alive incl. internal ${aliveAllNouns} + vfs file nouns ${model.vfsFileNouns} + vfs root baseline ${model.vfsBaselineNouns})` + ).toBe(expectedNounsAll) + expect( + ledger.verbs.all, + `[${label}] getCanonicalCounts().verbs.all mismatch: expected ${expectedVerbsAll} (alive relations ${aliveVerbs} + vfs contains verbs ${model.vfsContainsVerbs} + vfs root baseline ${model.vfsBaselineVerbs})` + ).toBe(expectedVerbsAll) + expect(ledger.nouns.counted, `[${label}] getCanonicalCounts().nouns.counted mismatch (should mirror getNounCount())`).toBe(expectedNounCount) + expect(ledger.verbs.counted, `[${label}] getCanonicalCounts().verbs.counted mismatch (should mirror getVerbCount())`).toBe(expectedVerbCount) + expect(ledger.suspect, `[${label}] getCanonicalCounts().suspect must be false — every delete in this biography proves its record`).toBe(false) +} diff --git a/tests/unit/brainy/lazy-notready-honor.test.ts b/tests/unit/brainy/lazy-notready-honor.test.ts index 4cfc6857..e53be4a6 100644 --- a/tests/unit/brainy/lazy-notready-honor.test.ts +++ b/tests/unit/brainy/lazy-notready-honor.test.ts @@ -1,38 +1,49 @@ /** * @module tests/unit/brainy/lazy-notready-honor * @description THE SILENT-EMPTY TRAP pin (found during a fleet adoption, - * SELF-ENGINE-PAIR-STANDARD): under `disableAutoRebuild: true`, the lazy + * SELF-ENGINE-PAIR-STANDARD): under `disableAutoRebuild: true`, the OLD lazy * first-query path (`ensureIndexesLoaded`) assessed ONLY the vector index's * readiness — a native METADATA provider reporting not-ready (its strand * report) never blocked the completion latch, so the promised lazy rebuild - * never fired and every `find()` silently returned `[]` on a populated - * store (measured: 52 entities durable-but-unqueryable, first query - * 0ms/0 rows). The law: a not-ready report from ANY provider falls through - * to the rebuild — never a silent empty. + * never fired and every `find()` silently returned `[]` on a populated store + * (measured: 52 entities durable-but-unqueryable, first query 0ms/0 rows). + * + * RE-POINTED to the health-gate law (a read never builds; a rebuild runs + * entirely at open): `ensureIndexesLoaded()` is now a pure CHECK. A not-ready + * report from ANY provider — metadata, vector, or graph — makes it THROW the + * matching typed `*NotReadyError` rather than silently letting the read + * proceed, and it NEVER calls `rebuildIndexesIfNeeded` (that is entirely + * open()'s job now — see the second describe block below). The spirit is + * unchanged: a not-ready report from any single provider can never be + * shadowed into a silent empty result. * * White-box provider-double pattern per tests/unit/brainy/migration-deference. */ import { describe, it, expect, afterEach, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy, MetadataIndexNotReadyError } from '../../../src/index.js' import { NounType } from '../../../src/types/graphTypes.js' import { createTestConfig } from '../../helpers/test-factory.js' interface BrainInternals { index: { size(): number } metadataIndex: { isReady?: () => boolean } - lazyRebuildCompleted: boolean - ensureIndexesLoaded(): Promise + ensureIndexesLoaded(): void rebuildIndexesIfNeeded(force?: boolean): Promise } const brains: Brainy[] = [] +const dirs: string[] = [] afterEach(async () => { for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) vi.restoreAllMocks() }) -async function warmLazyBrain(): Promise<{ brain: Brainy; internals: BrainInternals }> { +async function warmBrain(): Promise<{ brain: Brainy; internals: BrainInternals }> { const brain = new Brainy(createTestConfig({ disableAutoRebuild: true })) await brain.init() brains.push(brain) @@ -40,36 +51,59 @@ async function warmLazyBrain(): Promise<{ brain: Brainy; internals: BrainInterna await brain.add({ data: `row ${i}`, type: NounType.Document, metadata: { i } }) } const internals = brain as unknown as BrainInternals - internals.lazyRebuildCompleted = false // simulate the cold first query return { brain, internals } } -describe('lazy path honors EVERY provider’s not-ready report', () => { - it('a not-ready METADATA provider blocks the completion latch and fires the rebuild', async () => { - const { internals } = await warmLazyBrain() +describe('the read gate honors EVERY provider’s not-ready report', () => { + it('a not-ready METADATA provider refuses loudly — it never lets a read proceed, and it never rebuilds', async () => { + const { internals } = await warmBrain() // The trap's shape: vector side looks fine (populated), metadata - // provider says NOT ready — the old gate latched complete here. - ;(internals.metadataIndex as { isReady?: () => boolean }).isReady = () => false - const rebuildSpy = vi - .spyOn(internals, 'rebuildIndexesIfNeeded') - .mockResolvedValue(undefined) + // provider says NOT ready — the OLD gate silently latched complete here. + // The new gate refuses loudly instead; a read never triggers a rebuild. + internals.metadataIndex.isReady = () => false + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) - await internals.ensureIndexesLoaded() - - expect(rebuildSpy, 'not-ready metadata provider must fire the lazy rebuild').toHaveBeenCalledWith(true) + expect(() => internals.ensureIndexesLoaded()).toThrow(MetadataIndexNotReadyError) + expect(rebuildSpy, 'a read NEVER triggers a rebuild — building is entirely open()\'s job now').not.toHaveBeenCalled() }) - it('control: all providers ready/unknown+populated → latch completes, no rebuild', async () => { - const { internals } = await warmLazyBrain() - ;(internals.metadataIndex as { isReady?: () => boolean }).isReady = () => true - const rebuildSpy = vi - .spyOn(internals, 'rebuildIndexesIfNeeded') - .mockResolvedValue(undefined) - - await internals.ensureIndexesLoaded() + it('control: all providers ready/unknown+populated → the gate lets the read through, no rebuild', async () => { + const { internals } = await warmBrain() + internals.metadataIndex.isReady = () => true + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) + expect(() => internals.ensureIndexesLoaded()).not.toThrow() expect(rebuildSpy).not.toHaveBeenCalled() - expect(internals.lazyRebuildCompleted).toBe(true) }) }) + +describe('the open-time build honors the same law: a needed rebuild runs at open, never deferred to a read', () => { + it('disableAutoRebuild:true does not defer a needed rebuild past open() on a reopened, populated store', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-lazy-notready-honor-')) + dirs.push(dir) + + const writer = new Brainy(createTestConfig({ disableAutoRebuild: true, storage: { type: 'filesystem', path: dir } })) + await writer.init() + for (let i = 0; i < 3; i++) { + await writer.add({ data: `row ${i}`, type: NounType.Document, metadata: { i } }) + } + await writer.flush() + await writer.close() + + // Fresh instance over the same store: its derived indexes start empty in + // memory, so open()'s rebuildIndexesIfNeeded MUST fire (and complete) + // before init() returns — even though disableAutoRebuild is true, there + // is no first-query lazy path left to defer to. + const reader = new Brainy(createTestConfig({ disableAutoRebuild: true, storage: { type: 'filesystem', path: dir } })) + const internals = reader as unknown as { rebuildIndexesIfNeeded(force?: boolean): Promise } + const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded') + + await reader.init() + brains.push(reader) + + expect(rebuildSpy).toHaveBeenCalledTimes(1) + const rows = await reader.find({ where: { i: 1 } }) + expect(rows.length).toBe(1) + }, 30000) +}) diff --git a/tests/unit/brainy/metadata-provider-contract.test.ts b/tests/unit/brainy/metadata-provider-contract.test.ts index 7e690978..945c0670 100644 --- a/tests/unit/brainy/metadata-provider-contract.test.ts +++ b/tests/unit/brainy/metadata-provider-contract.test.ts @@ -1,15 +1,18 @@ /** * @module tests/unit/brainy/metadata-provider-contract - * @description Brainy-side wiring of the two metadata-provider contract additions - * confirmed with cor for the lockstep: + * @description Brainy-side wiring of the metadata-provider contract. * - * 1. `probeConsistency()` — an OPTIONAL O(1) cold-open consistency sampler. On the - * first read, brainy calls it once; on `false` it self-heals via - * `detectAndRepairCorruption()` (the metadata counterpart of the graph cold-load - * guard). The native provider implements it; the JS index omits it (no-op). - * 2. `getIdsForFilter(filter, opts?)` — brainy passes a page bound on the UNSORTED - * `find({ type, where, limit })` path so a native provider can early-stop. The JS - * index ignores `opts`. + * `getIdsForFilter(filter, opts?)` — brainy passes a page bound on the UNSORTED + * `find({ type, where, limit })` path so a native provider can early-stop. The JS + * index ignores `opts`. + * + * RETIRED (health-gate law): `probeConsistency()` / `ensureMetadataConsistencyProbed()` + * — a read-time consistency probe that launches `detectAndRepairCorruption()` on + * `false` was exactly the read-triggered dark rebuild the law forbids (a read must + * never start a store walk or a rebuild). The probe's diagnostic value lives on in + * `validateIndexConsistency()` / `repairIndex()`, which remain explicit, operator-invoked + * calls. The pin below confirms the retirement: `probeConsistency()` is never called by + * a read, even when a provider exposes it. * * These are unit tests of brainy's CALL behaviour (the real end-to-end honoring is * exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS @@ -19,7 +22,7 @@ import { describe, it, expect, beforeEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' -describe('metadata-provider contract wiring (probeConsistency + getIdsForFilter opts)', () => { +describe('metadata-provider contract wiring (getIdsForFilter opts)', () => { let brain: Brainy let mi: any @@ -29,47 +32,23 @@ describe('metadata-provider contract wiring (probeConsistency + getIdsForFilter await brain.add({ data: 'a', type: NounType.Thing, metadata: { kind: 'x' } }) await brain.add({ data: 'b', type: NounType.Thing, metadata: { kind: 'y' } }) mi = (brain as any).metadataIndex - ;(brain as any)._metadataConsistencyProbed = false // reset the one-shot guard }) - it('calls probeConsistency once on cold open and self-heals via detectAndRepairCorruption on false', async () => { + it('RETIRED: a read never calls probeConsistency() / self-heals via detectAndRepairCorruption — that is the read-triggered dark rebuild the health-gate law forbids', async () => { let probes = 0 let repairs = 0 - mi.probeConsistency = async () => { probes++; return false } // corrupt → must repair + mi.probeConsistency = async () => { probes++; return false } // would-be corrupt signal const origRepair = mi.detectAndRepairCorruption.bind(mi) mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } await brain.find({ where: { kind: 'x' } }) - expect(probes).toBe(1) - expect(repairs).toBe(1) - - // Second read must NOT re-probe (once per brain). await brain.find({ where: { kind: 'y' } }) - expect(probes).toBe(1) - expect(repairs).toBe(1) - }) - it('does NOT repair when the probe reports healthy', async () => { - let repairs = 0 - mi.probeConsistency = async () => true // clean - const origRepair = mi.detectAndRepairCorruption.bind(mi) - mi.detectAndRepairCorruption = async () => { repairs++; return origRepair() } + expect(probes).toBe(0) // no read-time probe exists anymore + expect(repairs).toBe(0) // and therefore no read-triggered self-heal either - await brain.find({ where: { kind: 'x' } }) - expect(repairs).toBe(0) - }) - - it('a probe failure never breaks the read (best-effort, retried next time)', async () => { - let probes = 0 - mi.probeConsistency = async () => { probes++; throw new Error('probe boom') } - - // The read still succeeds despite the throwing probe. - const rows = await brain.find({ where: { kind: 'x' } }) - expect(rows.length).toBe(1) - expect(probes).toBe(1) - // Guard reset on failure → the next read retries the probe. - await brain.find({ where: { kind: 'y' } }) - expect(probes).toBe(2) + delete mi.probeConsistency + mi.detectAndRepairCorruption = origRepair }) it('passes a page bound to getIdsForFilter on the unsorted find path (offset 0, brainy re-windows)', async () => { diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index 5968c620..31b9b216 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -25,7 +25,7 @@ */ import { describe, it, expect, afterEach, vi } from 'vitest' -import { Brainy } from '../../../src/index.js' +import { Brainy, VectorIndexNotReadyError } from '../../../src/index.js' import { NounType } from '../../../src/types/graphTypes.js' import { createTestConfig } from '../../helpers/test-factory.js' import { BaseStorage } from '../../../src/storage/baseStorage.js' @@ -43,9 +43,8 @@ interface BrainInternals { metadataIndex: { rebuild(...a: unknown[]): Promise } graphIndex: { size(): number; rebuild(...a: unknown[]): Promise } _indexEpochStale: boolean - lazyRebuildCompleted: boolean rebuildIndexesIfNeeded(force?: boolean): Promise - ensureIndexesLoaded(): Promise + ensureIndexesLoaded(): void storage: { readRawObject(p: string): Promise } } @@ -181,40 +180,40 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b expect(idxSpy).toHaveBeenCalledTimes(1) }) - // --- Hook 1: large-path first-query lazy force-rebuild deference ---------- + // --- Hook 1: read-gate deference (RE-POINTED — the health-gate law retired + // the first-query lazy force-rebuild entirely: ensureIndexesLoaded() is now + // a pure CHECK that never calls rebuildIndexesIfNeeded, migrating or not. + // What survives from the original law is the DEFERENCE itself: a migrating + // provider's report is never judged by the gate — it neither throws nor + // rebuilds — while the exact same not-ready report on a NON-migrating + // provider throws the typed error instead of ever rebuilding.) ------------ - it('lazy first-query force-rebuild is SKIPPED when the vector provider isMigrating()', async () => { - // disableAutoRebuild routes first queries through ensureIndexesLoaded() (the - // large-brain lazy path that would otherwise force a blocking rebuild). + it('the read gate defers to a migrating vector provider — a not-ready report neither throws nor rebuilds', async () => { const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) const internals = internalsOf(brain) const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) - // Simulate a cold/empty live vector index (cor is mid-swap, serving canonical). - vi.spyOn(internals.index, 'size').mockReturnValue(0) - internals.lazyRebuildCompleted = false + // Simulate a not-ready live vector index (cor is mid-swap, serving canonical). + ;(internals.index as unknown as { isReady?: () => boolean }).isReady = () => false setMigrating(internals.index, true) - await internals.ensureIndexesLoaded() - - // A query during cor's background swap must not trigger brainy's blocking rebuild. + expect(() => internals.ensureIndexesLoaded()).not.toThrow() + // A query during cor's background swap must not trigger brainy's own + // rebuild — reads never rebuild in any case, migrating or not. expect(rebuildSpy).toHaveBeenCalledTimes(0) }) - it('lazy first-query force-rebuild STILL fires when the vector provider is not migrating (control)', async () => { + it('the read gate THROWS for the same not-ready vector provider once migration clears (control)', async () => { const brain = await makeWarmBrain(2, { disableAutoRebuild: true }) const internals = internalsOf(brain) const rebuildSpy = vi.spyOn(internals, 'rebuildIndexesIfNeeded').mockResolvedValue(undefined) - vi.spyOn(internals.index, 'size').mockReturnValue(0) - internals.lazyRebuildCompleted = false + ;(internals.index as unknown as { isReady?: () => boolean }).isReady = () => false // No isMigrating → not deferring. - await internals.ensureIndexesLoaded() - - // Without deference, the cold empty index drives the lazy force-rebuild. - expect(rebuildSpy).toHaveBeenCalledTimes(1) - expect(rebuildSpy).toHaveBeenCalledWith(true) + expect(() => internals.ensureIndexesLoaded()).toThrow(VectorIndexNotReadyError) + // Still never rebuilds — the gate refuses loudly instead. + expect(rebuildSpy).toHaveBeenCalledTimes(0) }) // --- Hook 2: public stampBrainFormat() ----------------------------------- diff --git a/tests/unit/metadata-cold-read-guard.test.ts b/tests/unit/metadata-cold-read-guard.test.ts index 40d37de6..b4f82f15 100644 --- a/tests/unit/metadata-cold-read-guard.test.ts +++ b/tests/unit/metadata-cold-read-guard.test.ts @@ -3,10 +3,14 @@ * reported cold `find({ where })` returning a silent `[]` on a freshly-opened * brain (a native metadata index that reports data but has not loaded its field * postings). This guard, the field-index counterpart of verifyGraphAdjacencyLive, - * probes a known persisted value on the first filtered find(): if the index does - * not serve it, brainy rebuilds and re-probes, and raises a loud - * MetadataIndexNotReadyError only if the rebuild still can't serve — never a - * silent empty result that misrepresents existing data. + * probes a known persisted value on the first filtered find(). + * + * RE-POINTED to the health-gate law: the guard NEVER rebuilds and NEVER walks + * the store from a read — a read-path rebuild is exactly the dark-rebuild + * failure mode the law retires (open() alone owns building). When the probe + * cannot serve the known value it raises a loud MetadataIndexNotReadyError + * IMMEDIATELY, with no rebuild attempt in between — never a silent empty + * result that misrepresents existing data. * * The 8.0 JS index cold-loads correctly, so we simulate the cold native failure * mode by intercepting the provider's getIdsForFilter/rebuild. @@ -42,37 +46,19 @@ describe('Metadata cold-read guard (#venue silent-[])', () => { mi.rebuild = origRebuild }) - it('cold index: verifyMetadataLive self-heals via rebuild — find({where}) is correct, NOT silent []', async () => { + it('cold index: verifyMetadataLive REFUSES immediately — find({where}) throws MetadataIndexNotReadyError, NEVER a silent [], and NEVER a rebuild attempt', async () => { const mi = brain.metadataIndex const origGetIds = mi.getIdsForFilter.bind(mi) + let rebuilds = 0 const origRebuild = mi.rebuild.bind(mi) - let cold = true brain._metadataVerified = false // re-arm the one-shot for this scenario - mi.getIdsForFilter = async (...a: any[]) => (cold ? [] : origGetIds(...a)) - mi.rebuild = async () => { - await origRebuild() - cold = false // the rebuild warms the postings - } - try { - const res = await brain.find({ where: { status: 'active' }, limit: 100 }) - expect(res.length).toBe(1) // self-healed — the known entity is returned - } finally { - mi.getIdsForFilter = origGetIds - mi.rebuild = origRebuild - } - }) - - it('unrecoverably cold index: find({where}) throws MetadataIndexNotReadyError — never a silent []', async () => { - const mi = brain.metadataIndex - const origGetIds = mi.getIdsForFilter.bind(mi) - const origRebuild = mi.rebuild.bind(mi) - brain._metadataVerified = false - mi.getIdsForFilter = async () => [] // always cold; rebuild can't fix it - mi.rebuild = async () => {} + mi.getIdsForFilter = async () => [] // cold: the known value never resolves + mi.rebuild = async () => { rebuilds++; return origRebuild() } try { await expect(brain.find({ where: { status: 'active' }, limit: 100 })).rejects.toBeInstanceOf( MetadataIndexNotReadyError ) + expect(rebuilds).toBe(0) // the guard never rebuilds from a read — it refuses loudly instead } finally { mi.getIdsForFilter = origGetIds mi.rebuild = origRebuild diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index c43b2cf2..d4d268ac 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -63,6 +63,9 @@ function inGate(rel: string): boolean { return ( rel.startsWith('tests/unit/') || rel.startsWith('tests/integration/') || + // The lifecycle biography lane — included by the integration config + // ('tests/lifecycle/**/*.test.ts'; see tests/lifecycle/README.md). + rel.startsWith('tests/lifecycle/') || rel.endsWith('.unit.test.ts') || rel.endsWith('.integration.test.ts') ) diff --git a/tests/unit/utils/indexReadiness.test.ts b/tests/unit/utils/indexReadiness.test.ts new file mode 100644 index 00000000..e00d1ff0 --- /dev/null +++ b/tests/unit/utils/indexReadiness.test.ts @@ -0,0 +1,153 @@ +/** + * @module tests/unit/utils/indexReadiness + * @description Pins for the read-gate authority, {@link assessProviderHealth}, and + * its older sibling {@link assessIndexReadiness}. The health-gate law: a provider's + * NAMED, synchronous, O(1) health report — when exposed — REPLACES the `isReady()`/ + * size-heuristic fallback as the read gate's source of truth. A throw from + * `healthReport()` is a CONTRACT VIOLATION (never read as healthy, never swallowed + * into "unknown"); an `unledgered` family is UNKNOWN (never healthy, never broken — + * `serving` is always the provider's own verdict, verbatim). + */ +import { describe, it, expect } from 'vitest' +import { assessIndexReadiness, assessProviderHealth } from '../../../src/utils/indexReadiness.js' +import type { HealthReport, LedgerInvariantResult } from '../../../src/plugin.js' + +function invariant(overrides: Partial = {}): LedgerInvariantResult { + return { + name: 'manifest-residency', + holds: true, + detail: 'ok', + heal: 'none', + source: 'ledger', + ...overrides + } +} + +function report(overrides: Partial = {}): HealthReport { + return { + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 1, + generation: 1, + unledgered: [], + ...overrides + } +} + +describe('assessIndexReadiness (legacy isReady() classifier)', () => { + it('unknown when the provider is null/undefined', () => { + expect(assessIndexReadiness(null)).toBe('unknown') + expect(assessIndexReadiness(undefined)).toBe('unknown') + }) + + it('unknown when isReady() is absent', () => { + expect(assessIndexReadiness({})).toBe('unknown') + }) + + it('ready / not-ready mirror isReady()', () => { + expect(assessIndexReadiness({ isReady: () => true })).toBe('ready') + expect(assessIndexReadiness({ isReady: () => false })).toBe('not-ready') + }) +}) + +describe('assessProviderHealth — the read-gate authority', () => { + it('via "none": no provider at all', () => { + const a = assessProviderHealth(null) + expect(a.via).toBe('none') + expect(a.readiness).toBe('unknown') + expect(a.report).toBeNull() + expect(a.reasons.length).toBeGreaterThan(0) + }) + + it('via "size-heuristic": provider exposes neither healthReport() nor isReady()', () => { + const a = assessProviderHealth({}) + expect(a.via).toBe('size-heuristic') + expect(a.readiness).toBe('unknown') + expect(a.report).toBeNull() + }) + + it('via "is-ready": provider exposes isReady() but no healthReport() — ready', () => { + const a = assessProviderHealth({ isReady: () => true }) + expect(a.via).toBe('is-ready') + expect(a.readiness).toBe('ready') + expect(a.reasons).toEqual([]) + }) + + it('via "is-ready": isReady() === false — not-ready with a reason', () => { + const a = assessProviderHealth({ isReady: () => false }) + expect(a.via).toBe('is-ready') + expect(a.readiness).toBe('not-ready') + expect(a.reasons.length).toBeGreaterThan(0) + }) + + it('healthReport() present REPLACES isReady() — serving:true wins even if isReady() lies false', () => { + const p = { isReady: () => false, healthReport: () => report({ serving: true }) } + const a = assessProviderHealth(p) + expect(a.via).toBe('health-report') + expect(a.readiness).toBe('ready') + }) + + it('serving:true, healthy:true, no invariants failing → ready, no reasons', () => { + const p = { healthReport: () => report({ serving: true, healthy: true }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('ready') + expect(a.reasons).toEqual([]) + expect(a.report).toEqual(report({ serving: true, healthy: true })) + }) + + it('serving:false with a named heal:"rebuild" failing invariant → not-ready, reason names it', () => { + const failing = invariant({ name: 'posted-count-floor', holds: false, heal: 'rebuild', detail: 'posted 10 < canonical 20' }) + const p = { healthReport: () => report({ serving: false, healthy: false, invariants: [failing] }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('not-ready') + expect(a.reasons.some((r) => r.includes('posted-count-floor') && r.includes('heal:rebuild') && r.includes('posted 10 < canonical 20'))).toBe(true) + }) + + it('unledgered-only report (serving:true, no failing invariant) → ready, reason names the unledgered family', () => { + const p = { healthReport: () => report({ serving: true, healthy: true, unledgered: ['canonical-verb-coverage'] }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('ready') + expect(a.reasons.some((r) => r.includes('unledgered') && r.includes('canonical-verb-coverage'))).toBe(true) + }) + + it('UNLEDGERED IS UNKNOWN: an unledgered family never flips a NOT-serving provider to ready', () => { + const failing = invariant({ holds: false, heal: 'rebuild', name: 'x' }) + const p = { healthReport: () => report({ serving: false, healthy: false, invariants: [failing], unledgered: ['some-family'] }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('not-ready') + }) + + it('serving:true, healthy:false with a heal:"repair" failure → still ready (degraded-but-serving)', () => { + const failing = invariant({ name: 'stale-counter', holds: false, heal: 'repair', detail: 'counter drift' }) + const p = { healthReport: () => report({ serving: true, healthy: false, invariants: [failing] }) } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('ready') + expect(a.reasons.some((r) => r.includes('stale-counter') && r.includes('heal:repair'))).toBe(true) + }) + + it('healthReport() that THROWS is a CONTRACT VIOLATION: not-ready, via health-report, reason names the throw — never "unknown"', () => { + const p = { healthReport: () => { throw new Error('mmap window busy') } } + const a = assessProviderHealth(p) + expect(a.via).toBe('health-report') + expect(a.readiness).toBe('not-ready') + expect(a.report).toBeNull() + expect(a.reasons.some((r) => r.includes('mmap window busy'))).toBe(true) + expect(a.readiness).not.toBe('unknown') + }) + + it('healthReport() that throws a non-Error value still produces a named reason (String(err))', () => { + const p = { healthReport: () => { throw 'boom' } } + const a = assessProviderHealth(p) + expect(a.readiness).toBe('not-ready') + expect(a.reasons.some((r) => r.includes('boom'))).toBe(true) + }) + + it('the returned report carries the generation for narration dedup', () => { + const p = { healthReport: () => report({ generation: 42 }) } + const a = assessProviderHealth(p) + expect(a.report?.generation).toBe(42) + }) +}) diff --git a/tests/unit/vector-cold-read-guard.test.ts b/tests/unit/vector-cold-read-guard.test.ts index 49ca6426..0905f298 100644 --- a/tests/unit/vector-cold-read-guard.test.ts +++ b/tests/unit/vector-cold-read-guard.test.ts @@ -3,9 +3,14 @@ * @description Pattern-A / Finding 1: a pure semantic find({ query }) has no * filter, so verifyMetadataLive never fires — nothing guarded the vector index. * A cold native vector index that loaded its COUNT but not its serving structure - * returned a silent []. verifyVectorLive() closes that: honest isReady() first, - * else a known-vector self-match probe; self-heal (rebuild) or throw - * VectorIndexNotReadyError — never a silent empty result. + * returned a silent []. verifyVectorLive() closes that: the health-report/isReady() + * authority first, else a known-vector self-match probe. + * + * RE-POINTED to the health-gate law: the guard NEVER rebuilds and NEVER walks + * the store from a read — a read-path rebuild is exactly the dark-rebuild + * failure mode the law retires (open() alone owns building). A not-serving + * signal (from either strategy) THROWS VectorIndexNotReadyError immediately, + * with no rebuild attempt in between — never a silent empty result. */ import { describe, it, expect, beforeEach } from 'vitest' import { Brainy, NounType, VectorIndexNotReadyError } from '../../src/index.js' @@ -34,50 +39,37 @@ describe('Vector cold-read guard (verifyVectorLive) — silent-[] on cold semant vi.rebuild = origRebuild }) - it('cold index: verifyVectorLive self-heals via rebuild — semantic find is correct, NOT silent []', async () => { - const vi = brain.index - const origSearch = vi.search.bind(vi) - const origRebuild = vi.rebuild.bind(vi) - let cold = true - brain._vectorVerified = false - // size()>0 (count present) but search returns nothing until a rebuild warms it. - vi.search = async (...a: any[]) => (cold ? [] : origSearch(...a)) - vi.rebuild = async (...a: any[]) => { await origRebuild(...a); cold = false } - try { - const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) - expect(res.length).toBeGreaterThan(0) // self-healed - } finally { - vi.search = origSearch; vi.rebuild = origRebuild - } - }) - - it('unrecoverably cold index: semantic find throws VectorIndexNotReadyError', async () => { + it('cold index (no isReady()): verifyVectorLive REFUSES immediately — throws VectorIndexNotReadyError, NEVER rebuilds', async () => { const vi = brain.index const origSearch = vi.search.bind(vi) + let rebuilds = 0 const origRebuild = vi.rebuild.bind(vi) brain._vectorVerified = false - vi.search = async () => [] // always cold; rebuild can't fix it - vi.rebuild = async () => {} + // size()>0 (count present) but search never returns a hit for the known vector. + vi.search = async () => [] + vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } try { await expect( brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) ).rejects.toBeInstanceOf(VectorIndexNotReadyError) + expect(rebuilds).toBe(0) // the guard never rebuilds from a read — it refuses loudly instead } finally { vi.search = origSearch; vi.rebuild = origRebuild } }) - it('native provider reporting isReady()===false rebuilds, then serves', async () => { + it('native provider reporting isReady()===false THROWS immediately — never rebuilds', async () => { const vi = brain.index + let rebuilds = 0 const origRebuild = vi.rebuild.bind(vi) - let ready = false brain._vectorVerified = false - vi.isReady = () => ready - vi.rebuild = async (...a: any[]) => { await origRebuild(...a); ready = true } + vi.isReady = () => false + vi.rebuild = async (...a: any[]) => { rebuilds++; return origRebuild(...a) } try { - const res = await brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) - expect(ready).toBe(true) // rebuild ran because isReady() was false - expect(res).toBeDefined() + await expect( + brain.find({ query: 'x', searchMode: 'semantic', limit: 100 }) + ).rejects.toBeInstanceOf(VectorIndexNotReadyError) + expect(rebuilds).toBe(0) // a not-ready report throws immediately — it is never a rebuild trigger } finally { delete vi.isReady; vi.rebuild = origRebuild } From 18f172e0981286479c3e95731aa9e84cacead2ff Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 10:01:56 -0700 Subject: [PATCH 008/150] feat(recovery): the catchup verdict is consumed; verb rows go live; the metadata rebuild goes online MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three cures on the JS metadata index, one seam: - THE CATCHUP WIRING. The index computed its three-way watermark verdict at open and nothing consumed it — after a crash + adopt reopen, find() served the pre-crash index while canonical reads and counts recovered (caught by the lifecycle lane's first run). The open path now consumes the verdict: 'adopt' is a no-op, 'catchup' folds the fact window (stamped, committed] through the index legs — nouns and verbs, remove-then-add, one mechanism for add and update — and 'rescan' runs the explicit rebuild, each narrated. The lane's Ch4–6 release-blocking marker comes off: the contract holds. Bonus root-cause: close() never stamped the projection watermarks (only flush() did), so any close without a prior flush verdicted a needless 'rescan' on reopen — both doors now stamp. - THE LIVE VERB PATH. Verb rows entered the metadata index only via rebuild walks, so every rebuilt store minted phantom/stale verb postings from its first live relate(). relate()/unrelate()/updateRelation() and remove()'s cascade now post/retract the verb's row in the same commit as the graph leg — transact() planners mirror identically — using the exact record shape the rebuild walk uses, so live and rebuilt populations agree. - THE ONLINE REBUILD. rebuild() was clear-then-walk — every metadata read empty for the duration. rebuildMetadataIndexOnline builds a fresh manager beside the serving one (shared identity, in-memory build, dual-write via a shadow seam with zero call-site changes), atomically swaps the reference, and persists exactly once post-swap. A find() polled ~200x during a 2k-noun rebuild never dropped below its baseline. repairIndex({ rebuild: ['metadata'] }) uses it automatically. --- src/brainy.ts | 476 +++++++++++++++--- src/utils/metadataIndex.ts | 382 +++++++++++++- .../metadata-online-rebuild.test.ts | 167 ++++++ tests/integration/verb-metadata-rows.test.ts | 190 +++++++ tests/lifecycle/biography.test.ts | 27 +- .../utils/metadataIndex-watermark.test.ts | 150 +++++- 6 files changed, 1275 insertions(+), 117 deletions(-) create mode 100644 tests/integration/metadata-online-rebuild.test.ts create mode 100644 tests/integration/verb-metadata-rows.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index c8912e26..151f5f7b 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1385,6 +1385,20 @@ export class Brainy implements BrainyInterface { ]) } + // METADATA WATERMARK CATCHUP: the JS metadata index computed its + // three-way watermark verdict inside metadataIndex.init() above, + // against the generation store's now-FINAL committed generation (the + // crash-recovery fold above — the durable-at-ack replay of acked + // writes whose canonical bytes hadn't reached disk — has already run, + // and any rolled-back-transaction rebuild just above already brought + // every index current, so the verdict is consumed here whether or not + // that rebuild ran). Consumed BEFORE the rebuild gate below and BEFORE + // this open serves any read — the cure for the class of bug where + // canonical get()/counts recover a crash-window write but find() + // keeps serving the metadata index's pre-crash state (the index + // flushes only periodically, not per-commit). + await this.consumeMetadataWatermarkVerdict(generationOpenResult.rolledBackGenerations > 0) + // 8.0 versioned-provider replay-gap check: a provider whose persisted // index generation is behind the storage layer's committed generation // replays the gap itself (post-commit applier contract) — surface the @@ -3672,6 +3686,85 @@ export class Brainy implements BrainyInterface { if (deferringEmbed) this.kickEmbedWorker() } + /** + * @description Build the metadata-index retraction operation for one id + * (noun or verb) — the null-metadata-safe closure shared by every removal + * leg that reaches the metadata index with a possibly-missed pre-read: + * `remove()`'s own noun leg, its verb-cascade retractions, `unrelate()`, + * and their `transact()`/`planTx*` mirrors (both callers add the returned + * operation to their own batch — `tx.addOperation()` for a single-op + * transaction, `plan.operations.push()` for a planned `transact()` batch). + * THE NULL-METADATA SKIP IS CLOSED (a posting-leak class): + * - metadata present → the ordinary, provider-agnostic + * `RemoveFromMetadataIndexOperation` (exact per-field retraction). + * - metadata absent (a torn pre-read, or the row was already gone) → + * a provider exposing `removeEntityById` (the id-keyed contract) gets + * exact per-entity retraction via its reverse record; the JS index + * gets `removeFromIndex(id)` — safe id-keyed cleanup (deleted bitmap + + * id mapper; field statistics reconcile at the next rebuild/repairIndex), + * narrated; a native provider WITHOUT the contract is never called + * metadata-omitted (that path walks its value space) — the skip is + * tracked in the degraded set instead, narrated, so `repairIndex()` + * reconciles it (and this method returns `null` — no operation to add). + * Silence is the only thing outlawed. + * @param id - The noun/verb id being retracted. + * @param metadata - The pre-read metadata/entity structure, or falsy when + * the read missed. + * @param context - Narration prefix identifying the caller/id, e.g. + * `remove(${id})` or `remove(${entityId}) cascade unrelate ${verbId}`. + * @returns The operation to add to the caller's batch, or `null` when + * nothing could be done (already narrated + tracked as degraded). + */ + private metadataIndexRetractionOp( + id: string, + metadata: unknown, + context: string + ): Operation | null { + if (metadata) { + return new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration) + } + const prov = this.metadataIndex as unknown as { + removeEntityById?: (id: string) => Promise + removeFromIndex?: (id: string, metadata?: unknown, generation?: bigint) => Promise + } + if (typeof prov.removeEntityById === 'function') { + const g = this.indexWriteGeneration + return { + name: 'RemoveEntityByIdTombstone', + execute: async () => { + await prov.removeEntityById!(id) + return async () => { + // Undo of an id-keyed tombstone on an absent row: nothing to + // restore (the row had no readable metadata to re-post). + void g + } + } + } + } else if (this.metadataIndex instanceof MetadataIndexManager) { + const gv = this.indexWriteGeneration + prodLog.warn( + `[Brainy] ${context}: no metadata at delete — id-keyed index cleanup ran ` + + `(deleted bitmap + id mapper); field statistics reconcile at the next rebuild/repairIndex.` + ) + return { + name: 'IdKeyedIndexCleanup', + execute: async () => { + await prov.removeFromIndex!(id, undefined, typeof gv === 'function' ? gv() : gv) + return async () => {} + } + } + } else { + this._indexDegradedIds.add(id) + prodLog.warn( + `[Brainy] ${context}: no metadata at delete and this provider has no id-keyed ` + + `removal — its postings for this id are NOT tombstoned yet (tracked as degraded; ` + + `repairIndex() reconciles). Never calling a metadata-omitted native removal: that ` + + `path walks the store's value space.` + ) + return null + } + } + /** * Remove an entity and all its relationships * @@ -3736,61 +3829,11 @@ export class Brainy implements BrainyInterface { ) } - // Operation 2: Remove from metadata index. THE NULL-METADATA SKIP IS - // CLOSED (a posting-leak class, confirmed at this site): when the - // pre-read missed, the leg no longer silently skips — - // - a provider exposing removeEntityById (the id-keyed contract) - // gets it: exact per-entity retraction via its reverse record; - // - the JS index gets removeFromIndex(id) — safe id-keyed cleanup - // (deleted bitmap + id mapper; field stats reconcile at rebuild); - // - a NATIVE provider WITHOUT the contract is never called - // metadata-omitted (that path walks its value space) — the skip - // happens, but NARRATED and tracked in the degraded set so - // repairIndex reconciles it. Silence is the only thing outlawed. - if (metadata) { - tx.addOperation( - new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration) - ) - } else { - const prov = this.metadataIndex as unknown as { - removeEntityById?: (id: string) => Promise - removeFromIndex?: (id: string, metadata?: unknown, generation?: bigint) => Promise - } - if (typeof prov.removeEntityById === 'function') { - const g = this.indexWriteGeneration - tx.addOperation({ - name: 'RemoveEntityByIdTombstone', - execute: async () => { - await prov.removeEntityById!(id) - return async () => { - // Undo of an id-keyed tombstone on an absent row: nothing - // to restore (the row had no readable metadata to re-post). - void g - } - } - }) - } else if (this.metadataIndex instanceof MetadataIndexManager) { - const gv = this.indexWriteGeneration - tx.addOperation({ - name: 'IdKeyedIndexCleanup', - execute: async () => { - await prov.removeFromIndex!(id, undefined, typeof gv === 'function' ? gv() : gv) - return async () => {} - } - }) - prodLog.warn( - `[Brainy] remove(${id}): no metadata at delete — id-keyed index cleanup ran ` + - `(deleted bitmap + id mapper); field statistics reconcile at the next rebuild/repairIndex.` - ) - } else { - this._indexDegradedIds.add(id) - prodLog.warn( - `[Brainy] remove(${id}): no metadata at delete and this provider has no id-keyed ` + - `removal — its postings for this id are NOT tombstoned yet (tracked as degraded; ` + - `repairIndex() reconciles). Never calling a metadata-omitted native removal: that ` + - `path walks the store's value space.` - ) - } + // Operation 2: Remove from metadata index (null-metadata-safe — see + // metadataIndexRetractionOp's JSDoc for the full closure). + { + const retractionOp = this.metadataIndexRetractionOp(id, metadata, `remove(${id})`) + if (retractionOp) tx.addOperation(retractionOp) } // Operation 3: Delete noun (full removal). The pre-read metadata rides @@ -3808,6 +3851,21 @@ export class Brainy implements BrainyInterface { tx.addOperation( new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration) ) + // Retract the cascaded relation's metadata-index row too — the + // live mirror of what a rebuild would derive for this (now-gone) + // edge (mirrors the noun leg above). The whole hydrated verb + // (system fields top-level + the custom bag under `metadata`, + // same shape `extractIndexableFields` reads for any entity-record + // frame) is the before-image — every entry in `allVerbs` was + // already successfully hydrated by the reads above, so this is + // never metadata-omitted in practice, but the closure stays + // defensive rather than assuming. + { + const cascadeRetractionOp = this.metadataIndexRetractionOp( + verb.id, verb, `remove(${id}) cascade unrelate ${verb.id}` + ) + if (cascadeRetractionOp) tx.addOperation(cascadeRetractionOp) + } // Delete verb metadata tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) @@ -4641,6 +4699,16 @@ export class Brainy implements BrainyInterface { ) ) + // Operation 3b: Add the verb's metadata-index row, in the SAME + // commit as the graph leg — the live mirror of what rebuild()'s + // verb walk already derives (ADR-007 A4: one mechanism, never a + // second hand-rolled shape). `verbMetadata` is the exact raw stored + // record `SaveVerbMetadataOperation` above just persisted — the same + // shape `storage.getVerbMetadata()`/rebuild() read back. + tx.addOperation( + new AddToMetadataIndexOperation(this.metadataIndex, id, verbMetadata, this.indexWriteGeneration) + ) + // Create bidirectional if requested if (params.bidirectional && reverseId) { const reverseVerb: GraphVerb = { @@ -4678,6 +4746,13 @@ export class Brainy implements BrainyInterface { (verbInt) => this.cacheVerbInt(verbInt, reverseId) ) ) + + // Operation 6b: Add the reverse edge's metadata-index row (same + // stored shape as the primary edge — SaveVerbMetadataOperation + // above persists the same `verbMetadata` object for both). + tx.addOperation( + new AddToMetadataIndexOperation(this.metadataIndex, reverseId, verbMetadata, this.indexWriteGeneration) + ) } }, undefined, @@ -4760,6 +4835,15 @@ export class Brainy implements BrainyInterface { ) } + // Operation 1b: Retract the verb's metadata-index row — the live + // mirror of remove()'s cascade leg (null-metadata-safe; see + // metadataIndexRetractionOp's JSDoc). Nothing to retract when the + // pre-read found no verb (already gone / never existed). + if (verb) { + const retractionOp = this.metadataIndexRetractionOp(id, verb, `unrelate(${id})`) + if (retractionOp) tx.addOperation(retractionOp) + } + // Operation 2: Delete verb metadata (which also deletes vector) tx.addOperation( new DeleteVerbMetadataOperation(this.storage, id) @@ -4903,6 +4987,23 @@ export class Brainy implements BrainyInterface { new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata) ) + // Re-post the verb's metadata-index row — remove the old shape, add + // the new one, same commit (the plain pair; there is no update-op + // capability for the metadata leg yet — see the GRAPH leg's + // typeChanged branch just below for the capability this ISN'T: + // that's the graph adjacency's own remove+add, keyed on the verb + // TYPE changing; the metadata row updates on EVERY updateRelation() + // call, since metadata/subtype/weight/etc. can all change without a + // type change). `existing` is the pre-update hydrated verb (already + // read above); `updatedMetadata` is the raw stored record just + // persisted — the same shape relate()/rebuild() use to add. + tx.addOperation( + new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, existing, this.indexWriteGeneration) + ) + tx.addOperation( + new AddToMetadataIndexOperation(this.metadataIndex, params.id, updatedMetadata, this.indexWriteGeneration) + ) + // If the verb type changed, re-index in graph adjacency so traversal-by-type // stays consistent. The id is preserved across the swap. if (typeChanged && reindexInts) { @@ -10603,6 +10704,15 @@ export class Brainy implements BrainyInterface { new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration), new DeleteVerbMetadataOperation(this.storage, verb.id) ) + // Retract the cascaded relation's metadata-index row too — the + // transact() mirror of remove()'s single-op cascade leg + // (null-metadata-safe; see metadataIndexRetractionOp's JSDoc). + { + const cascadeRetractionOp = this.metadataIndexRetractionOp( + verb.id, verb, `transact remove(${id}) cascade unrelate ${verb.id}` + ) + if (cascadeRetractionOp) plan.operations.push(cascadeRetractionOp) + } plan.touchedVerbs.push(verb.id) state.verbs.delete(verb.id) state.removedVerbs.add(verb.id) @@ -10769,7 +10879,10 @@ export class Brainy implements BrainyInterface { // id mapper to assign an int for an entity that did not exist yet. new AddToGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, id) - ) + ), + // The transact() mirror of relate()'s metadata-index leg — same + // commit as the graph leg, same raw stored shape. + new AddToMetadataIndexOperation(this.metadataIndex, id, verbMetadata, this.indexWriteGeneration) ) plan.touchedVerbs.push(id) state.verbs.set(id, verb) @@ -10811,7 +10924,8 @@ export class Brainy implements BrainyInterface { new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata), new AddToGraphIndexOperation(this.graphIndex, reverseVerb, () => this.resolveVerbEndpointInts(reverseVerb), this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, reverseId) - ) + ), + new AddToMetadataIndexOperation(this.metadataIndex, reverseId, verbMetadata, this.indexWriteGeneration) ) plan.touchedVerbs.push(reverseId) state.verbs.set(reverseId, reverseVerb) @@ -10856,6 +10970,12 @@ export class Brainy implements BrainyInterface { // may have been created earlier in this same batch (forward refs). new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration) ) + // The transact() mirror of unrelate()'s metadata-index leg + // (null-metadata-safe; see metadataIndexRetractionOp's JSDoc — a + // present `verb` here is never metadata-omitted, but the closure + // stays defensive rather than assuming). + const retractionOp = this.metadataIndexRetractionOp(id, verb, `transact unrelate(${id})`) + if (retractionOp) plan.operations.push(retractionOp) } plan.operations.push(new DeleteVerbMetadataOperation(this.storage, id)) plan.touchedVerbs.push(id) @@ -11564,6 +11684,31 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Stamp every projection's watermark with the store's + * current committed generation — the door BOTH {@link flush} and {@link + * close} open right before persisting, so EITHER path leaves a stamped, + * `'adopt'`-verdicting artifact on disk (stamp-after-data still holds + * inside each owner: this only hands the generation over — the owner's + * OWN flush is what durably writes the stamp, LAST). Before this method + * existed, `close()` had its own separate flush fan-out that never + * stamped, so a `close()` without a preceding explicit `flush()` left + * every projection unstamped — a real, closed store that legitimately + * verdicts `'rescan'` on its very next open (not a bug in the verdict, + * a gap in `close()`'s persistence completeness that this closes). + * No `committedGeneration` capability, or a replacement provider that + * doesn't carry the stamp method (a native pair swaps these managers) = + * no stamp = the owner's verdict machinery treats the artifact as + * legacy — never a flush/close crash either way. + */ + private stampProjectionWatermarks(): void { + const wmGen = this.storage?.committedGeneration?.() ?? null + if (wmGen === null) return + ;(this.metadataIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) + ;(this.index as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) + ;(this.graphIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) + } + /** * Flush all indexes and caches to persistent storage * CRITICAL FIX: Ensures data survives server restarts @@ -11603,22 +11748,8 @@ export class Brainy implements BrainyInterface { await this.generationStore.flushPendingSingleOps() // Flush all components in parallel for performance - // Watermark stamps ride every flush fan-out: stamp each projection with - // the committed generation BEFORE its flush persists (stamp-after-data - // holds inside each owner — the stamp is its LAST write; here we only - // hand the generation over). No committedGeneration capability = no - // stamp = the owner's verdict machinery treats the artifact as legacy. - { - const wmGen = this.storage?.committedGeneration?.() ?? null - if (wmGen !== null) { - // ALL THREE optional-chained: a replacement provider (the native - // pair swaps these managers) may not carry the stamp method — a - // missing stamp is a verdict-side rescan, never a flush crash. - ;(this.metadataIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - ;(this.index as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - ;(this.graphIndex as { stampWatermark?: (g: number) => void }).stampWatermark?.(wmGen) - } - } + // Watermark stamps ride every flush fan-out — see stampProjectionWatermarks(). + this.stampProjectionWatermarks() await Promise.all([ // 1. Flush storage adapter counts (entity/verb counts by type) (async () => { @@ -16316,6 +16447,179 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Consume the JS metadata index's watermark verdict (see + * {@link MetadataIndexManager.watermarkVerdict}) at open — the coordinator + * half of the catchup wiring; {@link MetadataIndexManager.applyWatermarkCatchup} + * is the mechanism half. Feature-detected to the JS manager only: a native + * metadata-index provider consumes the same verdict door in its own train + * (this method never touches the native-provider wrapper contract). + * + * Ordering: called from `performInit()` immediately after + * `metadataIndex.init()` has computed the verdict against the generation + * store's now-FINAL committed generation, and BEFORE `rebuildIndexesIfNeeded()` + * (the open-time rebuild gate) or any read serves — so a caller can never + * observe the pre-catchup state. + * + * @param alreadyRebuilt - `true` when crash recovery just rebuilt every + * index from canonical (rolled-back uncommitted transactions) — the + * verdict's prescribed action is redundant with what already ran (a + * fresh canonical walk supersedes any catchup fold or rescan), so it is + * skipped, narrated, rather than duplicating the work. + */ + private async consumeMetadataWatermarkVerdict(alreadyRebuilt: boolean): Promise { + if (!(this.metadataIndex instanceof MetadataIndexManager)) return + const verdict = this.metadataIndex.watermarkVerdict() + if (verdict === null || verdict === 'adopt') return + + if (alreadyRebuilt) { + prodLog.info( + `[Brainy] metadata index watermark verdict '${verdict}' at open — skipped: crash ` + + `recovery already rebuilt every index from canonical this open.` + ) + return + } + + const window = this.metadataIndex.watermarkGap() + // A genuine first boot (no persisted artifact at all) verdicts 'rescan' + // too — same as a real unverifiable artifact — but it is routine, not + // alarming: narrate it at info level instead of warn (mirrors the + // manager's own internal distinction in loadWatermarkVerdict()). + const firstBoot = verdict === 'rescan' && !this.metadataIndex.watermarkArtifactPresent() + const preNarrate = firstBoot ? prodLog.info.bind(prodLog) : prodLog.warn.bind(prodLog) + preNarrate( + verdict === 'catchup' && window + ? `[Brainy] metadata index watermark verdict: CATCHUP — folding generations ` + + `(${window.from}, ${window.to}] from the fact log before this open serves reads.` + : firstBoot + ? `[Brainy] metadata index watermark verdict: rescan (no persisted artifact — first ` + + `boot; the rebuild below is a trivial no-op walk).` + : `[Brainy] metadata index watermark verdict: RESCAN — the persisted artifact is ` + + `unverifiable (unstamped, or ahead of the store's committed generation); ` + + `forcing a full rebuild from canonical at open.` + ) + + const scan = window + ? this.scanFacts({ fromGeneration: window.from + 1, toGeneration: window.to }) + : null + const result = await this.metadataIndex.applyWatermarkCatchup(scan) + + if (result.action === 'rescan') { + const postNarrate = firstBoot ? prodLog.debug.bind(prodLog) : prodLog.warn.bind(prodLog) + postNarrate( + `[Brainy] metadata index catchup demoted to a full rebuild` + + `${result.reason ? ` — ${result.reason}` : ''}.` + ) + } else if (result.action === 'caught-up') { + prodLog.warn( + `[Brainy] metadata index catchup complete: ${result.factsApplied} fact(s) folded ` + + `(${result.nounsApplied} noun op(s), ${result.verbsApplied} verb op(s)) — index now ` + + `reflects generation ${result.window?.to}.` + ) + } + } + + /** + * @description B3 Deliverable 3 — THE ONLINE METADATA REBUILD. + * `repairIndex()`'s ceremony door for the `'metadata'` family routes here + * instead of calling `MetadataIndexManager.rebuild()` directly: build a + * FRESH replacement manager BESIDE the live one (same storage, same + * idMapper — identity is shared, never a second mapper), walk canonical + * into it while every live write during the build ALSO mirrors there + * (`MetadataIndexManager.beginShadow`), fold the generation window the + * walk may have read stale, then atomically swap this brain's reference — + * `this.metadataIndex` points at the OLD manager for the ENTIRE build, so + * every read in progress (and every read that starts before the swap + * line executes) keeps serving its full, unbuilt-adjacent population; + * nothing ever observes a half-built index. + * + * PERSISTENCE CHOICE (named per the B3 brief): the JS manager's persisted + * keys (field-index chunks, column-store segments, the watermark stamp, + * the id-mapper record) are GLOBAL per storage — not namespaced per + * manager instance — so two managers cannot safely persist independently + * mid-build (a segment-number race, a stamp race, an id-mapper reload + * that would discard the live manager's not-yet-flushed assignments — + * see `MetadataIndexManager.initForShadowBuild`'s JSDoc for the id-mapper + * hazard specifically). This build therefore PERSISTS ONLY AT SWAP: the + * shadow builds entirely in memory (`rebuild({ inMemoryOnly: true })` + + * a fact-log fold — neither touches storage) and flushes exactly once, + * after the swap, as the sole owner of the shared keys. + * + * FALLBACK: a store with no fact log (or a non-JS/native metadata + * provider — its own train owns its online-rebuild strategy) cannot + * safely bound "what landed during the walk"; this method falls back to + * the ORIGINAL blocking clear-then-walk `rebuild()`, narrated. + */ + private async rebuildMetadataIndexOnline(): Promise { + if (!(this.metadataIndex instanceof MetadataIndexManager)) { + // A registered provider (e.g. a native accelerator) may replace + // `this.metadataIndex` with a non-MetadataIndexManager object at + // runtime even though the field's declared type is the JS class — + // the cast mirrors the same reach-in used elsewhere in this file + // (e.g. checkHealth()'s `metadataProvider` locals) for exactly this. + const provider = this.metadataIndex as unknown as MetadataIndexProvider + await provider.rebuild() + return + } + + const committedAtStart = this.storage.committedGeneration?.() ?? null + const factLogAvailable = committedAtStart !== null && this.scanFacts() !== null + if (!factLogAvailable) { + prodLog.warn( + `[Brainy] repairIndex(): metadata rebuild — no fact log on this store, build-beside ` + + `is unavailable; falling back to the blocking rebuild (reads may serve a ` + + `partially-built index for its duration).` + ) + await this.metadataIndex.rebuild() + return + } + + prodLog.warn( + `[Brainy] repairIndex(): metadata rebuild — building a fresh replacement index BESIDE ` + + `the live one (reads keep serving the current index throughout); swapping in ` + + `atomically once it is caught up.` + ) + const startedAt = Date.now() + const oldManager = this.metadataIndex + const shadow = new MetadataIndexManager(this.storage, {}, { + entityIdMapper: oldManager.getIdMapper() + }) + + oldManager.beginShadow(shadow) + let committedAtSwap: number + try { + await shadow.buildBeside(committedAtStart!) + // Capture the true final generation right before the swap — a + // synchronous read, no `await` between here and the reference + // assignment below, so nothing can land ungoverned in the gap: the + // shadow has been live-mirroring every write since beginShadow() + // above, and this generation is the floor a FUTURE open's watermark + // verdict will trust once stamped. + committedAtSwap = this.storage.committedGeneration?.() ?? committedAtStart! + } catch (err) { + oldManager.endShadow() + prodLog.error( + `[Brainy] repairIndex(): online metadata rebuild FAILED during the walk/fold — the ` + + `live index is UNCHANGED (never swapped); reads keep serving the current ` + + `(pre-rebuild) metadata index. Error: ${(err as Error).message}` + ) + throw err + } + + oldManager.endShadow() + this.metadataIndex = shadow + + // NOW persist — the shadow is the SOLE owner of the shared storage keys + // (nothing references `oldManager` any more; it never flushes again). + shadow.stampWatermark(committedAtSwap) + await shadow.flush() + + prodLog.warn( + `[Brainy] repairIndex(): online metadata rebuild complete in ${Date.now() - startedAt}ms — ` + + `swapped in a fresh index reflecting generation ${committedAtSwap}, zero read downtime.` + ) + } + /** * @description Rebuild indexes from persisted data if needed — THE OPEN-TIME * BUILD. Called once per open (init calls it; `repairIndex()`'s @@ -17189,7 +17493,15 @@ export class Brainy implements BrainyInterface { `[Brainy] repairIndex(): explicit rebuild requested for '${familyName}' — ` + `rebuilding unconditionally (no invariant consulted).` ) - await p.rebuild() + // The metadata family routes through the online build-beside + // orchestrator (B3 D3) instead of the provider's own rebuild() — + // zero read downtime when a fact log is available, narrated + // fallback to the blocking rebuild() otherwise. + if (familyName === 'metadata') { + await this.rebuildMetadataIndexOnline() + } else { + await p.rebuild() + } record(`provider:${familyName}`, { checked: true, healed: 1, @@ -17230,7 +17542,13 @@ export class Brainy implements BrainyInterface { `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + `requiring a rebuild — reconciling its derived state from canonical.` ) - await p.rebuild() + // See the explicit-rebuild branch above: 'metadata' routes through + // the online build-beside orchestrator (B3 D3). + if (familyName === 'metadata') { + await this.rebuildMetadataIndexOnline() + } else { + await p.rebuild() + } } else { record(`provider:${report.provider}`, { checked: true, healed: 0, @@ -17821,6 +18139,14 @@ export class Brainy implements BrainyInterface { } await this.autoCompactHistory() + // Watermark stamps ride this flush too — see stampProjectionWatermarks(). + // Read-only instances skip it (no writes, no committed-generation drift + // to certify; ensureInitialized()'s guard below never runs for them + // either, so this must not assume a writer's invariants). + if (!this.isReadOnly) { + this.stampProjectionWatermarks() + } + // Phase 1: Flush ALL components in parallel to persist buffered data // This is critical when cor native providers buffer data in Rust memory await Promise.all([ diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 13cf3bb4..3cc56b2e 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -20,6 +20,7 @@ import { type WatermarkVerdict, type WatermarkVerdictResult } from './projectionWatermark.js' +import type { FactScanHandle } from '../db/factLog.js' import { NounType, VerbType, @@ -77,6 +78,31 @@ export interface MetadataIndexStats { indexSize: number // in bytes } +/** + * @description What {@link MetadataIndexManager.applyWatermarkCatchup} did, + * for the caller's narration. + * - `'noop'` — the verdict was `null`/`'adopt'`: the artifact already + * reflects committed truth. Zero index writes. + * - `'rescan'` — the verdict was `'rescan'`, OR a `'catchup'` verdict was + * demoted (no window, or no fact log to scan) — either way a full + * {@link MetadataIndexManager.rebuild} already ran; `reason` names why. + * - `'caught-up'` — the `(from, to]` window folded successfully; the + * artifact is stamped and flushed at `to`. + */ +export interface CatchupApplyResult { + action: 'noop' | 'rescan' | 'caught-up' + /** Present on `'rescan'` — why the fold could not proceed as a catchup. */ + reason?: string + /** Present on `'caught-up'` — the fact-log window that was folded. */ + window?: { from: number; to: number } + /** Present on `'caught-up'` — noun ops applied (add/update/delete). */ + nounsApplied?: number + /** Present on `'caught-up'` — verb ops applied (add/update/delete). */ + verbsApplied?: number + /** Present on `'caught-up'` — distinct committed generations folded. */ + factsApplied?: number +} + export interface MetadataIndexConfig { maxIndexSize?: number // Max number of entries per field value (default: 10000) rebuildThreshold?: number // Rebuild if index is this % stale (default: 0.1) @@ -147,6 +173,52 @@ export class MetadataIndexManager implements MetadataIndexProvider { private stampedWatermark: number | null = null /** The three-way verdict computed at init; null until init runs. */ private loadVerdict: WatermarkVerdictResult | null = null + /** + * Set only when {@link loadVerdict}.verdict is `'rescan'`: whether a + * persisted artifact existed at load (even an unstamped/unverifiable + * one) — distinguishes genuine first boot (nothing here yet, routine) + * from an artifact whose watermark is unverifiable (the loud case). The + * verdict value alone doesn't carry this distinction; see {@link + * watermarkArtifactPresent}. + */ + private rescanArtifactPresent = false + + /** + * @description THE BUILD-BESIDE SEAM (B3 Deliverable 3): when set (via + * {@link beginShadow}), every live `addToIndex`/`removeFromIndex` call on + * THIS instance also applies to the shadow instance — so a caller building + * a fresh replacement manager beside this one (walking canonical into it) + * never misses a write that lands during the build. This is the ONE seam + * that makes build-beside possible without touching every call site: every + * existing `AddToMetadataIndexOperation`/`RemoveFromMetadataIndexOperation` + * (and the JS manager's own `rebuild()`/catchup fold) keep calling the SAME + * serving instance exactly as before; only THIS instance knows it is also + * mirroring to a shadow. Null = no build in flight (the overwhelmingly + * common case; the check costs one property read per write). + */ + private shadow: MetadataIndexManager | null = null + + /** + * @description Start mirroring every `addToIndex`/`removeFromIndex` call on + * this instance to `shadow` too — see {@link shadow}'s JSDoc. The caller + * owns sequencing: writes mirrored WHILE a canonical walk is populating + * `shadow` may be clobbered by the walk's own (possibly stale) reads for + * the same id; the caller closes that window with a bounded fact-log fold + * AFTER the walk (the same mechanism {@link applyWatermarkCatchup} uses) + * before treating `shadow` as authoritative. + * @param shadow - The manager to mirror writes to. + */ + beginShadow(shadow: MetadataIndexManager): void { + this.shadow = shadow + } + + /** + * @description Stop mirroring writes to a shadow (see {@link beginShadow}). + * Idempotent; a no-op when no shadow is attached. + */ + endShadow(): void { + this.shadow = null + } // Cardinality and field statistics tracking private fieldStats = new Map() @@ -1604,6 +1676,15 @@ export class MetadataIndexManager implements MetadataIndexProvider { for (const { field } of fields) { this.metadataCache.invalidatePattern(`field_values_${field}`) } + + // THE BUILD-BESIDE SEAM — see `shadow`'s JSDoc. Mirrors this write to a + // shadow manager under construction, if one is attached. `skipFlush: + // true` always: the shadow's own persistence is the build orchestrator's + // job (it flushes once, after the swap — never mid-build, to avoid + // colliding with this instance's own persisted keys). + if (this.shadow) { + await this.shadow.addToIndex(id, entityOrMetadata, true, false, generation) + } } /** @@ -1676,6 +1757,11 @@ export class MetadataIndexManager implements MetadataIndexProvider { // the real commit watermark (the JS mapper ignores it). this.idMapper.remove(id, generation) await this.idMapper.flush() + + // THE BUILD-BESIDE SEAM — see `shadow`'s JSDoc. + if (this.shadow) { + await this.shadow.removeFromIndex(id, metadata, generation) + } } /** @@ -2759,8 +2845,8 @@ export class MetadataIndexManager implements MetadataIndexProvider { * `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped < * committed; the gap from {@link watermarkGap} awaits an incremental * fold), `'rescan'` (unstamped or stamped above committed — never - * trusted). Null until init() has run. Computed and exposed only; no - * load behavior changes ride on it yet. + * trusted). Null until init() has run. The coordinator (`Brainy.open()`) + * consumes this via {@link applyWatermarkCatchup} right after init. */ watermarkVerdict(): WatermarkVerdict | null { return this.loadVerdict?.verdict ?? null @@ -2774,6 +2860,223 @@ export class MetadataIndexManager implements MetadataIndexProvider { return this.loadVerdict?.gap ?? null } + /** + * @description Meaningful only when {@link watermarkVerdict} is + * `'rescan'`: `true` when a persisted artifact existed at load (even an + * unstamped/unverifiable one — real prior state, worth narrating loudly); + * `false` for a genuine first boot (nothing persisted yet — a caller + * should narrate this at a routine log level, not as an alarm, even + * though the verdict value is the same `'rescan'` either way). + */ + watermarkArtifactPresent(): boolean { + return this.rescanArtifactPresent + } + + /** + * @description Consume the three-way watermark verdict {@link + * watermarkVerdict} computed at init — the cure for a crash-recovered + * store whose canonical reads/counts recover every acked write but whose + * metadata projection (flushed only periodically, not per-commit) keeps + * serving the pre-crash state. Call once, right after `init()`, before + * anything reads from this projection. + * + * - `null`/`'adopt'` → the artifact already reflects the store's + * committed generation. Zero index writes. + * - `'catchup'` → the caller-supplied `scan` (expected already opened + * over `(watermarkGap().from, watermarkGap().to]`) is folded in, ONE + * op at a time, through the SAME two legs {@link rebuild} uses (ADR-007 + * A4 — one mechanism, never a second hand-rolled add/update shape): a + * tombstone (`op.record === null`) retracts id-keyed (this projection + * keeps no per-record delta log, so the pre-crash metadata for that id + * — if any — is what a value-precise removal would need, and it isn't + * available; the same tradeoff `remove()`'s null-metadata closure + * already accepts elsewhere); an after-image retracts-then-reposts, so + * an update never leaves stale postings under the old field values. A + * fact outside the window is skipped defensively (belt: the scan is + * already opened to the window; suspenders: this loop never trusts an + * over-run). On success the artifact is stamped at `to` and flushed — + * the same STAMP-AFTER-DATA door {@link flush} always writes through. + * - `'rescan'` (or a `'catchup'` verdict with no window, or no `scan` to + * fold — the store hosts no fact log) → the persisted artifact is + * unverifiable; this method runs the existing {@link rebuild} itself + * rather than leave the caller to notice and trigger it separately. + * + * @param scan - An open fact scan covering the catchup window (see + * {@link Brainy.scanFacts}), or `null` when none is available/needed. + * Ignored when the verdict is not `'catchup'`. + * @returns What happened — see {@link CatchupApplyResult}. + */ + async applyWatermarkCatchup(scan: FactScanHandle | null): Promise { + const verdict = this.watermarkVerdict() + if (verdict === null || verdict === 'adopt') return { action: 'noop' } + + if (verdict === 'rescan') { + await this.rebuild() + return { + action: 'rescan', + reason: 'persisted artifact is unverifiable (unstamped, or stamped ABOVE the ' + + "store's committed generation) — never adopting unverifiable state" + } + } + + // verdict === 'catchup' + const window = this.watermarkGap() + if (window === null) { + await this.rebuild() + return { action: 'rescan', reason: "'catchup' verdict exposed no window — cannot bound a fold" } + } + if (scan === null) { + await this.rebuild() + return { + action: 'rescan', + reason: `no fact log available to fold the (${window.from}, ${window.to}] catchup window` + } + } + + const { nounsApplied, verbsApplied, factsApplied } = await this.foldFactWindow(scan, window.from, window.to) + + this.stampWatermark(window.to) + await this.flush() + return { action: 'caught-up', window, nounsApplied, verbsApplied, factsApplied } + } + + /** + * @description Fold an open fact scan's `(fromGeneration, toGeneration]` + * window into this projection, ONE op at a time, through the SAME two legs + * {@link rebuild} uses (ADR-007 A4 — one mechanism, never a second + * hand-rolled add/update shape): a tombstone retracts id-keyed; an + * after-image retracts-then-reposts. THE CORE LOOP shared by {@link + * applyWatermarkCatchup} (which stamps + flushes after) and {@link + * buildBeside} (which does neither — persistence is the caller's job, + * exactly once, after a swap). Never stamps, never flushes, never touches + * storage beyond what `addToIndex`/`removeFromIndex` do internally + * (skipFlush is always forced true). + * @param scan - An open fact scan. + * @param fromGeneration - Window lower bound (exclusive). + * @param toGeneration - Window upper bound (inclusive). + * @returns Counts for the caller's narration. + */ + private async foldFactWindow( + scan: FactScanHandle, + fromGeneration: number, + toGeneration: number + ): Promise<{ nounsApplied: number; verbsApplied: number; factsApplied: number }> { + let nounsApplied = 0 + let verbsApplied = 0 + let factsApplied = 0 + for await (const batch of scan.batches()) { + for (const fact of batch.facts) { + // Defensive containment: the scan is already opened to the window, + // but a fact outside it is never applied regardless. + if (fact.generation <= fromGeneration || fact.generation > toGeneration) continue + const generation = BigInt(fact.generation) + for (const op of fact.ops) { + if (op.record === null) { + // TOMBSTONE — the id-keyed removal path (no per-record delta + // log to recover the old field values from). + await this.removeFromIndex(op.id, undefined, generation) + } else { + // AFTER-IMAGE — retract any stale posting for this id, then + // repost the new shape. Covers both a fresh add (nothing to + // retract; a no-op-ish remove) and an update, through the same + // two calls. + await this.removeFromIndex(op.id, undefined, generation) + await this.indexStoredRecord(op.id, op.record.metadata, { + skipFlush: true, + deferWrites: false, + generation + }) + } + if (op.kind === 'noun') nounsApplied++ + else verbsApplied++ + } + factsApplied++ + } + } + return { nounsApplied, verbsApplied, factsApplied } + } + + /** + * @description B3 Deliverable 3 — the shadow-build lifecycle's init: the + * MINIMUM setup {@link buildBeside} needs, deliberately NOT the general + * {@link init} sequence. Two reasons general `init()` is unsafe for a + * build-beside shadow: + * 1. `init()` unconditionally re-initializes the id mapper from storage + * (`idMapper.init()`) — safe for a FRESH mapper, but this instance is + * constructed with the CURRENTLY-SERVING manager's SHARED, already-live + * mapper (identity is shared, never a second mapper — this train's own + * law). Re-running its init() would DISCARD every not-yet-flushed + * UUID↔int assignment sitting in memory, breaking the live manager's + * own serving mid-build. + * 2. `init()` loads the field registry and, on a registry that's + * missing/empty while canonical has entities (exactly the shape a + * rebuild is often invoked to FIX), triggers `rebuild()` itself — + * WITHOUT `inMemoryOnly`, which would touch the shared storage keys + * the live manager depends on. + * What this DOES run: the WASM roaring-bitmap library init (idempotent; + * needed before any column-store write) and the column store's OWN + * segment-manifest discovery (read-only against shared storage; needed so + * THIS instance's eventual post-swap flush continues segment numbering + * correctly instead of colliding with the retiring manager's segments). + */ + private async initForShadowBuild(): Promise { + await roaringLibraryInitialize() + try { + await this.columnStore.init(this.storage, this.idMapper) + } catch (err) { + prodLog.warn('[MetadataIndex] shadow build: column store storage discovery failed:', err) + } + } + + /** + * @description B3 Deliverable 3 — THE ONLINE REBUILD's manager-side half: + * populate THIS instance (expected fresh/empty, constructed with the SAME + * storage + idMapper as the manager it will replace — see {@link + * initForShadowBuild}) from canonical storage without ever touching the + * shared storage keys the currently-serving manager depends on — no chunk + * deletion, no flush, anywhere in this call. The caller (the brain's + * rebuild-beside orchestrator) is responsible for: + * 1. Attaching this instance as a {@link beginShadow} target on the OLD + * manager BEFORE calling this, so live writes during the walk mirror + * here too (best-effort — the walk below may still clobber a mirrored + * write with a stale read for the same id; the fold after the walk is + * what makes the final state authoritative, not the mirror). + * 2. Swapping its own reference to this instance once this resolves. + * 3. Calling {@link stampWatermark} + {@link flush} EXACTLY ONCE, after + * the swap — this instance never persists itself. + * @param committedGenerationAtStart - The store's committed generation + * captured by the caller BEFORE this call — the fold's lower bound. + * @returns The generation this instance's canonical data reflects once the + * walk + fold settle — the fold's upper bound (writes committed after + * this point but before the swap only reach this instance via the live + * {@link beginShadow} mirror, so the caller re-reads the store's + * committed generation right before stamping, rather than trusting this + * return value as final). + * @throws If canonical advanced during the walk but no fact log is + * available to fold the gap — never a silently incomplete shadow. + */ + async buildBeside(committedGenerationAtStart: number): Promise { + await this.initForShadowBuild() + await this.rebuild({ inMemoryOnly: true }) + + const committedAfterWalk = this.storage.committedGeneration?.() ?? committedGenerationAtStart + if (committedAfterWalk > committedGenerationAtStart) { + const scan = this.storage.scanFacts?.({ + fromGeneration: committedGenerationAtStart + 1, + toGeneration: committedAfterWalk + }) ?? null + if (scan === null) { + throw new Error( + `MetadataIndexManager.buildBeside: canonical advanced from generation ` + + `${committedGenerationAtStart} to ${committedAfterWalk} during the walk, but this ` + + `store hosts no fact log to fold the gap — refusing a silently incomplete shadow` + ) + } + await this.foldFactWindow(scan, committedGenerationAtStart, committedAfterWalk) + } + return committedAfterWalk + } + /** * @description Write the pending watermark stamp as a sidecar record — * always called AFTER the data it certifies is durable. A stamp-write @@ -2826,6 +3129,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { if (result.verdict === 'rescan') { const artifactPresent = this.fieldIndexes.size > 0 || stamped !== null + this.rescanArtifactPresent = artifactPresent if (artifactPresent) { prodLog.warn( `[MetadataIndex] watermark verdict: RESCAN — persisted index is ` + @@ -3484,13 +3788,48 @@ export class MetadataIndexManager implements MetadataIndexProvider { } } + /** + * @description Index one raw stored noun/verb record — THE ONE add leg + * shared by {@link rebuild}'s canonical walk and {@link + * applyWatermarkCatchup}'s fact-log fold (ADR-007 A4: one mechanism, + * never a second hand-rolled shape). No conversion step is needed here: + * a raw stored record (`storage.getNounMetadata`/`getVerbMetadata`, or a + * fact's after-image `record.metadata`) is byte-identical — both read the + * exact same canonical path — and already the v2 nested-bag + * ("entity-record") shape {@link extractIndexableFields} expects. + * @param id - Entity/relationship id. + * @param storedMetadata - The raw stored metadata record. + * @param opts.skipFlush - Forwarded to {@link addToIndex}. + * @param opts.deferWrites - Forwarded to {@link addToIndex}. + * @param opts.generation - Forwarded to {@link addToIndex}. + */ + private async indexStoredRecord( + id: string, + storedMetadata: unknown, + opts: { skipFlush: boolean; deferWrites: boolean; generation?: bigint } + ): Promise { + await this.addToIndex(id, storedMetadata, opts.skipFlush, opts.deferWrites, opts.generation) + } + /** * Rebuild entire index from scratch using pagination * Non-blocking version that yields control back to event loop * Sparse indices now lazy-loaded via UnifiedCache (no need to clear Map) + * + * @param options.inMemoryOnly - B3 Deliverable 3 (build-beside): when + * `true`, this call never touches the shared storage keys another, + * currently-serving `MetadataIndexManager` over the SAME storage may + * depend on — it skips deleting persisted legacy chunk files AND skips + * the final `flush()` (which would otherwise write field indexes AND + * flush the column store's tail buffers to shared segment keys, + * colliding with a live manager's own writes). The caller ({@link + * buildBeside}) owns persistence entirely — exactly once, after this + * instance becomes the sole owner via an atomic swap. Default `false` + * (every other caller keeps today's clear-then-persist behavior). */ - async rebuild(): Promise { + async rebuild(options?: { inMemoryOnly?: boolean }): Promise { if (this.isRebuilding) return + const inMemoryOnly = options?.inMemoryOnly ?? false this.isRebuilding = true try { @@ -3519,15 +3858,22 @@ export class MetadataIndexManager implements MetadataIndexProvider { // here — it's always saved at the end of rebuild via flush(). This ensures // that if rebuild fails partway, the next init() can still discover fields // and trigger another rebuild attempt. - prodLog.info('Clearing existing metadata index chunks from storage...') - const existingFields = await this.getPersistedFieldList() + // + // SKIPPED for inMemoryOnly: these are the SHARED storage keys a live + // manager over the same storage may still be reading (see this + // method's JSDoc) — deleting them before the swap is a live-read + // hazard, not a cleanup. + if (!inMemoryOnly) { + prodLog.info('Clearing existing metadata index chunks from storage...') + const existingFields = await this.getPersistedFieldList() - if (existingFields.length > 0) { - for (const field of existingFields) { - await this.deleteFieldChunks(field) + if (existingFields.length > 0) { + for (const field of existingFields) { + await this.deleteFieldChunks(field) + } + + prodLog.info(`Cleared ${existingFields.length} field indexes from storage`) } - - prodLog.info(`Cleared ${existingFields.length} field indexes from storage`) } // EntityIdMapper is intentionally NOT cleared here. Rebuild re-iterates @@ -3582,7 +3928,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { for (const noun of result.items) { const metadata = metadataBatch.get(noun.id) if (metadata) { - await this.addToIndex(noun.id, metadata, true, true) + await this.indexStoredRecord(noun.id, metadata, { skipFlush: true, deferWrites: true }) } } @@ -3627,7 +3973,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { for (const verb of result.items) { const metadata = verbMetadataBatch.get(verb.id) if (metadata) { - await this.addToIndex(verb.id, metadata, true, true) + await this.indexStoredRecord(verb.id, metadata, { skipFlush: true, deferWrites: true }) } } @@ -3637,8 +3983,16 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Flush to storage. The column store's flush() handles tail-buffer-to- // segment promotion + manifest persistence. - prodLog.debug('💾 Flushing metadata index to storage...') - await this.flush() + // + // SKIPPED for inMemoryOnly — see this method's JSDoc: flush() writes + // the shared field-index keys AND flushes the column store's tail + // buffers to shared segment keys, which would race a live manager's + // own flushes over the SAME storage. The caller flushes exactly once, + // after the swap. + if (!inMemoryOnly) { + prodLog.debug('💾 Flushing metadata index to storage...') + await this.flush() + } prodLog.info(`✅ Metadata index rebuild completed! Processed ${totalNounsProcessed} nouns and ${totalVerbsProcessed} verbs`) diff --git a/tests/integration/metadata-online-rebuild.test.ts b/tests/integration/metadata-online-rebuild.test.ts new file mode 100644 index 00000000..bf7cebdb --- /dev/null +++ b/tests/integration/metadata-online-rebuild.test.ts @@ -0,0 +1,167 @@ +/** + * @module tests/integration/metadata-online-rebuild + * @description THE ONLINE JS METADATA REBUILD (B3 Deliverable 3) pins. + * `MetadataIndexManager.rebuild()` used to be clear-then-walk — reads went + * dark for the duration. `repairIndex({ rebuild: ['metadata'] })` now builds + * a fresh replacement index BESIDE the live one (walk canonical + mirror + * every live write via `beginShadow`/`endShadow` + a bounded fact-log fold), + * then atomically swaps the brain's reference — `find()` never observes a + * half-built index, and a write landing DURING the build is never lost. + */ +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import type { MetadataIndexManager } from '../../src/utils/metadataIndex.js' + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +function metadataIndexOf(brain: Brainy): MetadataIndexManager { + return (brain as unknown as { metadataIndex: MetadataIndexManager }).metadataIndex +} + +async function openBrain(): Promise<{ brain: Brainy; dir: string }> { + const dir = mkdtempSync(join(tmpdir(), 'brainy-online-rebuild-')) + dirs.push(dir) + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + persistence: { policy: 'manual' }, + logAuthority: 'adopt' + }) + await brain.init() + brains.push(brain) + return { brain, dir } +} + +describe('repairIndex({ rebuild: ["metadata"] }) — the online build-beside rebuild', () => { + it( + 'a find() polled throughout the rebuild of a 2k-noun store never returns fewer rows than ' + + 'before the build started, and a write landing DURING the build is never lost', + async () => { + const { brain, dir } = await openBrain() + void dir + + const N = 2000 + const ids: string[] = [] + for (let i = 0; i < N; i++) { + ids.push( + await brain.add({ + data: `entity ${i}`, + type: NounType.Person, + metadata: { status: i % 2 === 0 ? 'active' : 'inactive' } + }) + ) + } + for (let i = 0; i < 20; i++) { + await brain.relate({ + from: ids[i], to: ids[i + 1], type: VerbType.WorksWith, metadata: { tag: 'orig' } + }) + } + await brain.flush() + + const baseline = await brain.find({ where: { status: 'active' }, limit: 10000 }) + expect(baseline.length).toBe(N / 2) + + // Kick off the online rebuild WITHOUT awaiting — poll reads and + // perform a live write concurrently with it. + const repairPromise = brain.repairIndex({ rebuild: ['metadata'] }) + + let minObserved = Infinity + let polls = 0 + const pollPromise = (async () => { + // Poll until the rebuild settles — bounded so a slow CI box can't + // spin forever, generous enough to actually overlap the walk. + while (polls < 200) { + const rows = await brain.find({ where: { status: 'active' }, limit: 10000 }) + minObserved = Math.min(minObserved, rows.length) + polls++ + await new Promise((resolve) => setTimeout(resolve, 1)) + } + })() + + const newId = await brain.add({ + data: 'added during the rebuild', + type: NounType.Person, + metadata: { status: 'active' } + }) + const newRelId = await brain.relate({ + from: newId, to: ids[0], type: VerbType.WorksWith, metadata: { tag: 'during-build' } + }) + + const [report] = await Promise.all([repairPromise, pollPromise]) + + // THE PIN: never fewer rows than the pre-build baseline, at any polled + // instant — reads served the OLD (fully-populated) manager throughout. + expect(polls).toBeGreaterThan(0) + expect(minObserved).toBeGreaterThanOrEqual(baseline.length) + + // The repair report still accounts for the family (same receipt shape + // regardless of which rebuild mechanism actually ran underneath). + const metadataFamily = report.families.find((f) => f.family === 'provider:metadata') + expect(metadataFamily?.checked).toBe(true) + expect(metadataFamily?.rebuilt).toBe(true) + + // Post-swap correctness: the live write during the build was never + // lost (the beginShadow mirror + post-walk fold caught it). + const afterActive = await brain.find({ where: { status: 'active' }, limit: 10000 }) + expect(afterActive.length).toBe(baseline.length + 1) + expect(afterActive.some((r) => r.id === newId)).toBe(true) + + const index = metadataIndexOf(brain) + expect(await index.getIds('tag', 'during-build')).toEqual([newRelId]) + expect((await index.getIds('tag', 'orig')).length).toBe(20) + + // The swap stamped the watermark — a reopen adopts, zero rebuild. + await brain.close() + brains.length = 0 // already closed above; afterEach must not double-close + const reopened = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + persistence: { policy: 'manual' }, + logAuthority: 'adopt' + }) + await reopened.init() + brains.push(reopened) + const reopenedIndex = metadataIndexOf(reopened) + expect(reopenedIndex.watermarkVerdict()).toBe('adopt') + const reopenedActive = await reopened.find({ where: { status: 'active' }, limit: 10000 }) + expect(reopenedActive.length).toBe(afterActive.length) + }, + 60000 + ) + + it('repairIndex({ rebuild: ["metadata"] }) on an empty store is a trivial no-op walk', async () => { + const { brain } = await openBrain() + const report = await brain.repairIndex({ rebuild: ['metadata'] }) + const metadataFamily = report.families.find((f) => f.family === 'provider:metadata') + expect(metadataFamily?.checked).toBe(true) + expect(await brain.getNounCount()).toBe(0) + }) + + it('two consecutive online rebuilds both leave the index correct (idempotent)', async () => { + const { brain } = await openBrain() + const a = await brain.add({ data: 'a', type: NounType.Person, metadata: { status: 'active' } }) + await brain.add({ data: 'b', type: NounType.Person, metadata: { status: 'inactive' } }) + await brain.flush() + + await brain.repairIndex({ rebuild: ['metadata'] }) + const first = await brain.find({ where: { status: 'active' } }) + expect(first.map((r) => r.id)).toEqual([a]) + + await brain.repairIndex({ rebuild: ['metadata'] }) + const second = await brain.find({ where: { status: 'active' } }) + expect(second.map((r) => r.id)).toEqual([a]) + }) +}) diff --git a/tests/integration/verb-metadata-rows.test.ts b/tests/integration/verb-metadata-rows.test.ts new file mode 100644 index 00000000..79ce00d0 --- /dev/null +++ b/tests/integration/verb-metadata-rows.test.ts @@ -0,0 +1,190 @@ +/** + * @module tests/integration/verb-metadata-rows + * @description THE LIVE VERB PATH pins. Before this train, verb rows entered + * the metadata index ONLY via `MetadataIndexManager.rebuild()`'s canonical + * walk — every relate()/unrelate()/updateRelation() call, and every + * remove()-cascaded relationship, left the metadata index blind to verb + * writes until the next rebuild. This file pins that `relate()`, + * `unrelate()`, `updateRelation()`, `remove()`'s cascade, and their + * `transact()` mirrors now post/retract the SAME verb rows a rebuild would + * derive from canonical (ADR-007 A4: one mechanism for add/update, live and + * rebuilt). + */ +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import type { MetadataIndexManager } from '../../src/utils/metadataIndex.js' + +/** The JS metadata-index manager backing a memory-storage brain in these + * tests (feature-detected in production code via `instanceof + * MetadataIndexManager`; a narrow test-only reach-in here, matching the + * existing idiom in tests/integration/find-where-zero.test.ts and + * tests/integration/level-field-shadow.test.ts). */ +function metadataIndexOf(brain: Brainy): MetadataIndexManager { + return (brain as unknown as { metadataIndex: MetadataIndexManager }).metadataIndex +} + +describe('verb metadata rows — the live path matches the rebuild walk', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + async function addPerson(label: string): Promise { + return brain.add({ + data: `person ${label}`, + type: NounType.Person, + metadata: { label } + }) + } + + it('(a) relate() posts a metadata-index-backed verb row a query can find', async () => { + const a = await addPerson('a') + const b = await addPerson('b') + const relId = await brain.relate({ + from: a, to: b, type: VerbType.WorksWith, metadata: { role: 'lead' } + }) + + // Read it back the SAME way a rebuild-sourced row is queried — the + // manager's own posting lookup, keyed on the custom field the caller wrote. + const index = metadataIndexOf(brain) + expect(await index.getIds('role', 'lead')).toEqual([relId]) + }) + + it('(b) unrelate() retracts the row', async () => { + const a = await addPerson('a') + const b = await addPerson('b') + const relId = await brain.relate({ + from: a, to: b, type: VerbType.WorksWith, metadata: { role: 'lead' } + }) + + const index = metadataIndexOf(brain) + expect(await index.getIds('role', 'lead')).toEqual([relId]) + + // Flush BEFORE retracting the field's only occurrence: this durably + // persists the 'role' column (a segment on disk/in the store), so the + // post-retraction query below reads "this field exists, zero live + // postings" (→ []) rather than "this field has never been written" + // (→ FIELD_NOT_INDEXED) — an orthogonal column-store characteristic + // (an unflushed field with its last live posting removed reverts to + // unknown), not a D2 behavior. + await brain.flush() + + await brain.unrelate(relId) + + expect(await index.getIds('role', 'lead')).toEqual([]) + }) + + it('(c) updateRelation({ metadata }) leaves exactly the new values', async () => { + const a = await addPerson('a') + const b = await addPerson('b') + const relId = await brain.relate({ + from: a, to: b, type: VerbType.WorksWith, metadata: { role: 'lead', team: 'core' } + }) + + const index = metadataIndexOf(brain) + expect(await index.getIds('role', 'lead')).toEqual([relId]) + + // Flush first — see (b)'s note: 'role'/'team' must be durably known + // fields before their only value is retracted, or the post-update + // "gone" checks below throw FIELD_NOT_INDEXED instead of returning []. + await brain.flush() + + await brain.updateRelation({ id: relId, metadata: { role: 'reviewer' }, merge: false }) + + // Stale values gone (the old shape AND the merge:false-dropped field)… + expect(await index.getIds('role', 'lead')).toEqual([]) + expect(await index.getIds('team', 'core')).toEqual([]) + // …only the new value serves. + expect(await index.getIds('role', 'reviewer')).toEqual([relId]) + }) + + it("(d) remove(entity) cascade retracts every incident relation's metadata row", async () => { + const a = await addPerson('a') + const b = await addPerson('b') + const c = await addPerson('c') + const rel1 = await brain.relate({ + from: a, to: b, type: VerbType.WorksWith, metadata: { tag: 'cascade-test' } + }) + const rel2 = await brain.relate({ + from: c, to: a, type: VerbType.WorksWith, metadata: { tag: 'cascade-test' } + }) + + const index = metadataIndexOf(brain) + expect((await index.getIds('tag', 'cascade-test')).sort()).toEqual([rel1, rel2].sort()) + + // Flush first — see (b)'s note. + await brain.flush() + + await brain.remove(a) // a is source of rel1, target of rel2 — both cascade + + expect(await index.getIds('tag', 'cascade-test')).toEqual([]) + }) + + it('(e) a rebuild() reproduces exactly the verb-row population the live path built', async () => { + const a = await addPerson('a') + const b = await addPerson('b') + const c = await addPerson('c') + await brain.relate({ from: a, to: b, type: VerbType.WorksWith, metadata: { tag: 'parity', label: 'ab' } }) + await brain.relate({ from: b, to: c, type: VerbType.RelatedTo, metadata: { tag: 'parity', label: 'bc' } }) + const relId3 = await brain.relate({ + from: c, to: a, type: VerbType.WorksWith, metadata: { tag: 'parity', label: 'ca' } + }) + await brain.unrelate(relId3) // exercise retraction too — the rebuild must NOT resurrect it + + const index = metadataIndexOf(brain) + const beforeIds = (await index.getIds('tag', 'parity')).slice().sort() + expect(beforeIds.length).toBe(2) + const beforeAb = await index.getIds('label', 'ab') + const beforeBc = await index.getIds('label', 'bc') + + await index.rebuild() + + const afterIds = (await index.getIds('tag', 'parity')).slice().sort() + expect(afterIds).toEqual(beforeIds) + expect(await index.getIds('label', 'ab')).toEqual(beforeAb) + expect(await index.getIds('label', 'bc')).toEqual(beforeBc) + expect(await index.getIds('label', 'ca')).toEqual([]) // the unrelated edge stays gone + }) + + it('(f) transact() relate/unrelate posts/retracts the same metadata-index rows as single-op', async () => { + const a = await addPerson('a') + const b = await addPerson('b') + const c = await addPerson('c') + const d = await addPerson('d') + + // Single-op baseline. + const singleOpId = await brain.relate({ + from: a, to: b, type: VerbType.WorksWith, metadata: { tag: 'parity-f' } + }) + + // transact() mirror. + const relateDb = await brain.transact([ + { op: 'relate', from: c, to: d, type: VerbType.WorksWith, metadata: { tag: 'parity-f' } } + ]) + const transactId = relateDb.receipt!.ids[0] + await relateDb.release() + + const index = metadataIndexOf(brain) + expect((await index.getIds('tag', 'parity-f')).sort()).toEqual([singleOpId, transactId].sort()) + + // Flush first — see (b)'s note: 'tag' must be durably known before its + // last live posting is retracted below. + await brain.flush() + + // Retract both ways — single-op unrelate() and transact() unrelate. + await brain.unrelate(singleOpId) + const unrelateDb = await brain.transact([{ op: 'unrelate', id: transactId }]) + await unrelateDb.release() + + expect(await index.getIds('tag', 'parity-f')).toEqual([]) + }) +}) diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts index 305d7a99..9fda62a5 100644 --- a/tests/lifecycle/biography.test.ts +++ b/tests/lifecycle/biography.test.ts @@ -286,32 +286,7 @@ describe.sequential('lifecycle — the working store', () => { 300000 ) - /** - * Ch4 CRASH is a LIVE ENGINE FINDING, not a defect in this lane (see - * README.md and the project report this lane's build produced): after a - * crash (writes acked at commit but never flushed, the process abandoned - * exactly as `abandonAsCrashed` models, then reopened), canonical storage - * (`get()`), the vector index, and `getNounCount()`/`getCanonicalCounts()` - * all correctly recover every acked write — but the METADATA INDEX behind - * `find({ where })` recovers NONE of the crash-window's acked writes - * (neither new adds nor metadata updates to pre-existing entities), even - * though `getIndexStatus()` reports `projections.metadata.synchronous: - * true`. `repairIndex()` cannot close the gap either: its own report names - * `provider:metadata` as `checked: false, skipped: "no - * validateInvariants/rebuild contract"`. The assertion below states the - * TRUE contract (find() must agree with get()) and is expected to fail - * against the current engine — it must never be loosened to paper over - * this. Ch5/Ch6 are written in full below it and will start running the - * moment this gap is closed; they are not dead code, they are blocked code. - */ - // RELEASE-BLOCKING FINDING (the kill-matrix convention: assert the CONTRACT, - // mark `.fails`, never weaken): after a crash + adopt reopen, the JS metadata - // index computes its watermark verdict but nothing consumes 'catchup' - // (metadataIndex.ts loadWatermarkVerdict) — find() serves the pre-crash - // index while get()/counts recover. The catchup wiring is the cure; when it - // lands this `.fails` marker MUST be removed (vitest will force it: a - // passing `.fails` test is itself a failure). - it.fails( + it( 'Ch4 CRASH -> Ch5 REPAIR -> Ch6 SECOND LIFE: continues the Ch3 store', async () => { try { diff --git a/tests/unit/utils/metadataIndex-watermark.test.ts b/tests/unit/utils/metadataIndex-watermark.test.ts index 6c195b35..6d6f6e28 100644 --- a/tests/unit/utils/metadataIndex-watermark.test.ts +++ b/tests/unit/utils/metadataIndex-watermark.test.ts @@ -11,8 +11,11 @@ * Same rule, same verdict names as the shipped aggregation machinery * (AggregationIndex.stateAdoptionVerdict). * - * The verdict is COMPUTED AND EXPOSED only — these pins assert no rebuild - * trigger changed; acting on 'catchup' lands with the coordinator's wiring. + * The verdict is computed at init and consumed via + * {@link MetadataIndexManager.applyWatermarkCatchup} — the coordinator + * (`Brainy.performInit`) calls it right after `init()`, with an open fact + * scan when the verdict is `'catchup'`. This file pins both halves: the + * verdict computation (above) and the fold/no-op/demotion behavior below. */ import { describe, it, expect, vi, afterEach } from 'vitest' import { v4 as uuidv4 } from 'uuid' @@ -22,6 +25,46 @@ import { } from '../../../src/utils/metadataIndex.js' import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' import { prodLog } from '../../../src/utils/logger.js' +import type { CommitFact, FactScanBatch, FactScanHandle } from '../../../src/db/factLog.js' + +/** A fact scan handle over an in-memory list of facts — batches them one + * fact at a time (batch size is irrelevant to the fold, which reads + * `batch.facts` only). */ +function fakeScan(facts: CommitFact[]): FactScanHandle { + return { + headGeneration: facts.length > 0 ? facts[facts.length - 1].generation : 0, + segmentCount: 1, + approxFactCount: facts.length, + async *batches(): AsyncGenerator { + for (const fact of facts) { + yield { + facts: [fact], + firstGeneration: fact.generation, + lastGeneration: fact.generation, + factCount: 1, + byteSize: 0, + segmentId: 'fake' + } + } + }, + summary: () => ({ factsYielded: facts.length, segmentsRead: 1 }) + } +} + +/** One noun after-image fact — the flat-record shape (no nested `metadata` + * key), matching this file's existing `writeArtifact` convention. */ +function nounAdd(generation: number, id: string, metadata: Record): CommitFact { + return { + generation, + timestamp: Date.now(), + ops: [{ kind: 'noun', id, record: { metadata, vector: null } }] + } +} + +/** One noun tombstone fact. */ +function nounDelete(generation: number, id: string): CommitFact { + return { generation, timestamp: Date.now(), ops: [{ kind: 'noun', id, record: null }] } +} /** Fresh storage with a controllable committed generation. */ async function makeStorage(committed: number | null): Promise { @@ -169,3 +212,106 @@ describe('metadata index — watermark stamp + three-way load verdict', () => { expect(await storage.getMetadata(METADATA_INDEX_STAMP_KEY)).toBeNull() }) }) + +describe('metadata index — applyWatermarkCatchup (the coordinator door)', () => { + it("an 'adopt' verdict performs zero index writes", async () => { + const storage = await makeStorage(5) + await writeArtifact(storage, 5) + const index = await reopen(storage) + expect(index.watermarkVerdict()).toBe('adopt') + + const addSpy = vi.spyOn(index, 'addToIndex') + const removeSpy = vi.spyOn(index, 'removeFromIndex') + + const result = await index.applyWatermarkCatchup(null) + + expect(result).toEqual({ action: 'noop' }) + expect(addSpy).not.toHaveBeenCalled() + expect(removeSpy).not.toHaveBeenCalled() + }) + + it('a catchup window folding an add, an update (same id twice), and a delete → the index serves exactly the final state', async () => { + const storage = await makeStorage(5) + + // Session 1: two pre-existing entities, stamped at generation 5. + const survivorId = uuidv4() + const deletedId = uuidv4() + { + const index = new MetadataIndexManager(storage) + await index.init() + await index.addToIndex(survivorId, { status: 'active' }) + await index.addToIndex(deletedId, { status: 'active' }) + index.stampWatermark(5) + await index.flush() + } + + // The store advanced to generation 8 without another metadata flush — + // the exact shape a crash-then-adopt-reopen leaves behind. + setCommitted(storage, 8) + + const index = await reopen(storage) + expect(index.watermarkVerdict()).toBe('catchup') + expect(index.watermarkGap()).toEqual({ from: 5, to: 8 }) + + const addedId = uuidv4() + const scan = fakeScan([ + nounAdd(6, addedId, { status: 'new' }), // add + nounAdd(7, addedId, { status: 'updated' }), // update — same id twice + nounDelete(8, deletedId) // delete + ]) + + const result = await index.applyWatermarkCatchup(scan) + + expect(result.action).toBe('caught-up') + expect(result.window).toEqual({ from: 5, to: 8 }) + expect(result.factsApplied).toBe(3) + expect(result.nounsApplied).toBe(3) + expect(result.verbsApplied).toBe(0) + + // Final state: the added/updated id serves ONLY its final value... + expect(await index.getIds('status', 'updated')).toEqual([addedId]) + expect(await index.getIds('status', 'new')).toEqual([]) // stale value gone + // ...the deleted id is gone... + expect(await index.getIds('status', 'active')).toEqual([survivorId]) + // ...and the untouched survivor is unaffected. + expect(await index.getIds('status', 'active')).toContain(survivorId) + + // The window is certified: watermark stamped at `to`, and a fresh + // reopen now verdicts 'adopt'. + expect(index.watermark()).toBe(8) + const reopened = await reopen(storage) + expect(reopened.watermarkVerdict()).toBe('adopt') + }) + + it("a 'rescan' verdict runs the existing rebuild path instead of folding", async () => { + const storage = await makeStorage(9) + await writeArtifact(storage, 9) + setCommitted(storage, 4) // a truncated log pulled the watermark back — stamp ABOVE committed → rescan + + const index = await reopen(storage) + expect(index.watermarkVerdict()).toBe('rescan') + + const rebuildSpy = vi.spyOn(index, 'rebuild') + const result = await index.applyWatermarkCatchup(null) + + expect(result.action).toBe('rescan') + expect(result.reason).toBeTruthy() + expect(rebuildSpy).toHaveBeenCalledTimes(1) + }) + + it("a 'catchup' verdict with no fact log available demotes to rebuild, narrated", async () => { + const storage = await makeStorage(5) + await writeArtifact(storage, 5) + setCommitted(storage, 8) + + const index = await reopen(storage) + expect(index.watermarkVerdict()).toBe('catchup') + + const rebuildSpy = vi.spyOn(index, 'rebuild') + const result = await index.applyWatermarkCatchup(null) // no scan — no fact log + + expect(result.action).toBe('rescan') + expect(result.reason).toContain('no fact log') + expect(rebuildSpy).toHaveBeenCalledTimes(1) + }) +}) From b9ba50fbec4c697c8dffceedd8fd72ad95989287 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 10:01:56 -0700 Subject: [PATCH 009/150] =?UTF-8?q?fix(plugins):=20the=20silent-degrade=20?= =?UTF-8?q?doors=20close=20=E2=80=94=20a=20broken=20accelerator=20install?= =?UTF-8?q?=20can=20never=20read=20as=20absent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auto-detection "not installed" heuristic accepted any resolution failure whose message merely CONTAINED the package name, unterminated — so a missing platform-binary sibling package (what a deploy replacing node_modules mid-restart leaves behind) read as "the accelerator is not installed", and brainy silently served the default WASM engines with zero journal lines. A production restart storm paid 90 seconds of throttled WASM compile behind exactly that hole. The name must now terminate where it ends (quote, whitespace, punctuation, end) — a sibling package, an inner file path, or a dependency failure is a broken install and init() throws, as the guard's own law always stated. Second door: activate() returning false (the documented graceful decline) warned on console.warn, which `silent: true` patches away — an invisible degrade. The decline now narrates via the always-on channel. Also exports CanonicalCounts from the public surface (the coverage-ledger denominator type consumers read through getCanonicalCounts()). Pinned in tests/unit/plugin-activation-loudness.test.ts (five error shapes; the decline warn under silent: true). --- src/brainy.ts | 11 ++- src/index.ts | 5 +- src/plugin.ts | 11 ++- tests/unit/plugin-activation-loudness.test.ts | 71 +++++++++++++++++++ 4 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 tests/unit/plugin-activation-loudness.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 151f5f7b..5a600402 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -17622,8 +17622,15 @@ export class Brainy implements BrainyInterface { private static isPackageNotInstalledError(error: unknown, pkg: string): boolean { const code = (error as { code?: string })?.code const message = error instanceof Error ? error.message : String(error) - const namesPackage = - message.includes(`'${pkg}'`) || message.includes(`"${pkg}"`) || message.includes(` ${pkg}`) + // The package name must TERMINATE where it ends: an unanchored prefix match + // read a missing platform-binary SIBLING package (e.g. "-linux-x64-gnu", + // exactly what a deploy replacing node_modules mid-restart leaves behind) as + // " is not installed" — and a present-but-broken accelerator silently + // degraded to the default JS engines. A production storm was hunted for a + // day because of that swallow. The name must be followed by a quote, + // whitespace, punctuation, or end-of-message — never a longer name's tail. + const escaped = pkg.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + const namesPackage = new RegExp("(^|['\"\\s])" + escaped + "(?=$|['\"\\s.,)])").test(message) const isResolutionFailure = code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' || diff --git a/src/index.ts b/src/index.ts index 07f318c0..973136a5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -389,7 +389,10 @@ import type { HNSWVerb, HNSWConfig, StorageAdapter, - DerivedFamilyDeclaration + DerivedFamilyDeclaration, + // The canonical count ledger a storage adapter maintains (counted + ALL-visibility + // scalars per family, the coverage-ledger denominators) — see StorageAdapter.getCanonicalCounts. + CanonicalCounts } from './coreTypes.js' // Export vector index implementation (the JS HNSW path) diff --git a/src/plugin.ts b/src/plugin.ts index 47a559b6..bfdc403a 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -9,6 +9,7 @@ * registered manually via `brain.use()` — there is no implicit detection. */ +import { prodLog } from './utils/logger.js' import type { StorageAdapter, Vector, @@ -1574,9 +1575,13 @@ export class PluginRegistry { this.activated.add(name) activated.push(name) } else { - // Documented graceful decline (activate() → false). Surface it loudly so - // a silent degrade to the default engine never goes unnoticed. - console.warn( + // Documented graceful decline (activate() → false). Surface it on the + // ALWAYS-ON channel: `silent: true` patches console, and a declined + // accelerator warned into a patched console is a silent degrade to the + // default engines — the exact invisible-fallback class this registry + // exists to prevent (a production storm ran the WASM engine for 90s + // behind one suppressed warn). + prodLog.warn( `[brainy] Plugin "${name}" declined activation (activate() returned false); ` + `the default engine is in use for its providers.` ) diff --git a/tests/unit/plugin-activation-loudness.test.ts b/tests/unit/plugin-activation-loudness.test.ts new file mode 100644 index 00000000..23e50a50 --- /dev/null +++ b/tests/unit/plugin-activation-loudness.test.ts @@ -0,0 +1,71 @@ +/** + * @module tests/unit/plugin-activation-loudness + * @description The plugin-activation swallow closes. Two laws: + * (1) THE NOT-INSTALLED FREE PASS IS EXACT — a resolution failure earns the + * silent skip ONLY when it names the probed package itself, terminated + * where the name ends. A missing platform-binary SIBLING package + * ("-linux-x64-gnu" — what a deploy replacing node_modules + * mid-restart leaves), an inner file path, or a dependency failure is a + * BROKEN install and must fail loud. A production storm ran 90s of + * throttled WASM behind this exact prefix-match hole. + * (2) A GRACEFUL DECLINE IS NARRATED ON THE ALWAYS-ON CHANNEL — activate() + * returning false warns via prodLog, which `silent: true` cannot patch + * away; a declined accelerator is never an invisible degrade. + */ +import { describe, it, expect, vi, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { prodLog } from '../../src/utils/logger.js' + +const isNotInstalled = (error: unknown, pkg: string): boolean => + (Brainy as unknown as { + isPackageNotInstalledError(e: unknown, p: string): boolean + }).isPackageNotInstalledError(error, pkg) + +const resolutionError = (message: string): Error => { + const e = new Error(message) as Error & { code?: string } + e.code = 'ERR_MODULE_NOT_FOUND' + return e +} + +describe('the not-installed free pass is exact', () => { + const PKG = '@soulcraft/cor' + + it('the package itself, quoted or bare → not-installed (the one free path)', () => { + expect(isNotInstalled(resolutionError(`Cannot find package '${PKG}' imported from /app/x.js`), PKG)).toBe(true) + expect(isNotInstalled(resolutionError(`Cannot find module ${PKG}`), PKG)).toBe(true) + }) + + it('a missing platform-binary SIBLING package is a broken install, never not-installed', () => { + expect(isNotInstalled(resolutionError(`Cannot find package '${PKG}-linux-x64-gnu' imported from /app`), PKG)).toBe(false) + expect(isNotInstalled(resolutionError(`Failed to resolve ${PKG}-darwin-arm64`), PKG)).toBe(false) + }) + + it('an inner file path or a non-resolution error is never not-installed', () => { + expect(isNotInstalled(resolutionError(`Cannot find module '/app/node_modules/${PKG}/native/b.node'`), PKG)).toBe(false) + expect(isNotInstalled(new Error(`dlopen failed: wrong ELF class in ${PKG}`), PKG)).toBe(false) + }) +}) + +describe('a graceful decline is narrated on the always-on channel', () => { + afterEach(() => vi.restoreAllMocks()) + + it('activate() → false warns via prodLog even under silent: true', async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + const warn = vi.spyOn(prodLog, 'warn') + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true, + dimensions: 384 + }) + brain.use({ name: 'declining-accelerator', activate: async () => false }) + await brain.init() + try { + expect( + warn.mock.calls.some((c) => String(c[0]).includes('"declining-accelerator" declined activation')) + ).toBe(true) + } finally { + await brain.close().catch(() => {}) + } + }) +}) From 8cced871a02bff1a2f1a6a3154e646637a3c803e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 10:02:25 -0700 Subject: [PATCH 010/150] =?UTF-8?q?docs(release):=20the=2010.4.0=20entry,?= =?UTF-8?q?=20the=20index-health=20concept=20doc,=20and=20the=20API=20surf?= =?UTF-8?q?aces=20=E2=80=94=20written=20from=20the=20tree,=20not=20the=20p?= =?UTF-8?q?lan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 84 ++++++++ docs/PERFORMANCE.md | 71 ++---- docs/PLUGINS.md | 31 ++- docs/api/README.md | 103 +++++++++ docs/architecture/index-architecture.md | 8 + .../initialization-and-rebuild.md | 10 + docs/concepts/index-health.md | 204 ++++++++++++++++++ tests/lifecycle/README.md | 6 + 8 files changed, 454 insertions(+), 63 deletions(-) create mode 100644 docs/concepts/index-health.md diff --git a/RELEASES.md b/RELEASES.md index cc0272c3..45420bd1 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,90 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.4.0 — 2026-08-25 (the health report has a name) + +Three related cures, one root cause: an index deciding whether it could be trusted +by sampling itself instead of by exact accounting. This release replaces every +sampled self-probe with ledger-derived truth, and a read against an unhealthy index +now refuses loudly instead of guessing. + +- **The canonical count ledger.** Storage now tracks two scalars per family + (nouns/verbs) on the write path: the user-facing `counted` total — unchanged, + still what `getNounCount()` / `getVerbCount()` return — and a new ALL-visibility + `all` total covering every tier, the real denominator a derived index's own + coverage math needs. The unfiltered storage-level `totalCount` returned by + `getNouns()` / `getVerbs()` is now this unclamped ALL scalar; previously it could + only ever move up (`Math.max(scalar, scanned)`), so an inflated counter could + never self-correct. A delete that cannot prove the record it removed actually + existed (no canonical read, no prior image available) no longer decrements on + faith — it marks the ledger `suspect` (narrated once per session) instead of + silently drifting, and the next `repairIndex()` clears the flag with a real + recount. +- **One contract for a throwing health probe.** A provider's `validateInvariants()` + is documented to never throw — but if one does anyway (a bug, a transient fault), + it is now read the same way everywhere: `heal: 'none'`, the error named in the + report, never synthesized into a rebuild trigger and never swallowed into "looks + fine." A flaky check can no longer buy itself a rebuild. `repairIndex()`'s + per-family receipt also gains `missing` (an exact count plus a capped id sample), + `rebuilt` (a full rebuild ran, vs. an incremental heal), and `reason`. +- **The named health report; reads refuse instead of rebuilding.** Any index + provider may now expose a synchronous, O(1) `healthReport()` — composed from the + provider's own exact ledgers, never a sample — and this is the one signal + Brainy's read gate trusts. The first-query lazy-build path is gone: `brain.init()` + now runs every needed rebuild to completion before it returns, always, regardless + of dataset size. A read that lands on a provider whose health report says it + isn't serving throws a typed error instead of triggering a rebuild mid-query — + `GraphIndexNotReadyError`, `MetadataIndexNotReadyError`, or + `VectorIndexNotReadyError` (all exported from `@soulcraft/brainy`), naming the + reasons. `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] | 'all' })` is + the new explicit operator door: it rebuilds the named family unconditionally, no + health check consulted — reach for it when you have independent reason to + distrust a family regardless of what it self-reports. Bare `repairIndex()` is + unchanged in spirit: report-driven, heals only what its own checks say needs it. +- New concept doc: [Index Health](docs/concepts/index-health.md) walks the whole + story from a consumer's side — degraded-but-serving vs. not-ready, what + `repairIndex()` checks and heals per family, what `suspect` counts mean. + +**Nothing to change to adopt this.** No API removed, no signature narrowed — +`repairIndex()` gains an optional options bag and its return value gains fields, +both additive. The honest notes: if your code ever relied on a `find()` against a +cold/not-yet-built index quietly triggering a rebuild and returning results a beat +later, that behavior is gone — it now throws one of the three typed +`*NotReadyError` classes instead (catch them if you need to distinguish "not ready +yet" from "no results"). And `disableAutoRebuild: true` no longer defers index +construction to the first query — a needed rebuild always runs at `open()` now; +the flag has no effect on timing. Full manual control still lives in +`repairIndex({ rebuild: [...] })`. + +- **Crash-reopen catchup.** After an unclean shutdown, the metadata index now + folds the exact fact window it missed — `find()` serves every acked write on + reopen, closing the gap where canonical reads and counts recovered a + crash-window write but the index kept serving its pre-crash state until the + next full rebuild. Related root-cause fixed alongside: `close()` never + stamped the index watermarks (only `flush()` did), so a close without a + prior flush caused a needless full rescan verdict on the next open. +- **Relation rows are live in the metadata index.** Previously verb rows + entered the metadata index only during a rebuild — so a rebuilt store's + relation postings went stale from the first `relate()` after it. Relations + are now posted and retracted on the live write path (relate / unrelate / + updateRelation / remove's cascade, and their `transact()` forms), in the + same commit as the graph leg. +- **The metadata rebuild is online.** `rebuild()` for the metadata family no + longer clears and rebuilds in place (reads went empty for the duration): it + builds a complete replacement beside the serving index, mirrors concurrent + writes to both, swaps atomically, and persists once after the swap. Reads + never observe a partial index. `repairIndex({ rebuild: ['metadata'] })` uses + it automatically. +- **A broken accelerator install can never read as "not installed."** The + auto-detection free pass now requires the resolution error to name the + accelerator package itself, exactly — a missing platform-binary sibling + package, an inner file path, or a dependency failure is a broken install and + `init()` throws loudly. And a plugin that declines activation is narrated on + the always-on log channel, so `silent: true` can no longer hide a fallback + to the default engines. + +--- + ## v10.3.1 — 2026-08-18 (the fold that behaves) Three recovery cures from one production first-boot incident (a brain's first diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 248a2c70..b543e84a 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -323,58 +323,24 @@ Only the graph adjacency index carries a committed scale assertion: - ✅ **Single-Node by Design**: One process owns one `path`; scale out at the service layer - ✅ **Zero Stubs**: Every line of code is production-ready -## Lazy Loading Performance +## Index Build at Open (10.4+) -Brainy supports two initialization modes for optimal performance across different use cases: +As of 10.4, `brain.init()` runs every needed index rebuild to completion before +it returns — always, regardless of dataset size. There is no lazy, +first-query rebuild path: a brain either finishes opening healthy, or `init()` +fails loudly. `disableAutoRebuild` no longer defers index construction to a +first query; it has no effect on *when* a rebuild runs. Manual control over +rebuilds is `repairIndex({ rebuild: [...] })`. See +[Index Health](concepts/index-health.md) for the full read-gate contract +(providers self-report readiness via `healthReport()`; a read against a +not-serving provider throws a typed `*NotReadyError` rather than rebuilding +mid-query). -### Mode 1: Auto-Rebuild (Default) - -```javascript -const brain = new Brainy() -await brain.init() // Rebuilds indexes during init (~500ms-3s for 10K entities) -``` - -**Performance:** -- Init time: 500ms-3s (depends on dataset size) -- First query: Instant (indexes already loaded) -- Use case: Traditional applications, long-running servers - -### Mode 2: Lazy Loading - -```javascript -const brain = new Brainy({ disableAutoRebuild: true }) -await brain.init() // Returns instantly (0-10ms) - -const results = await brain.find({ limit: 10 }) // First query triggers rebuild (~50-200ms) -const more = await brain.find({ limit: 100 }) // Subsequent queries instant (0ms check) -``` - -**Performance:** -- Init time: 0-10ms (instant) -- First query: 50-200ms (includes index rebuild for 1K-10K entities) -- Subsequent queries: 0ms check (instant) -- Concurrent queries: Wait for same rebuild (mutex prevents duplicates) - -**Concurrency Safety:** -```javascript -// 100 concurrent queries immediately after init -await brain.init() - -const promises = Array.from({ length: 100 }, () => - brain.find({ limit: 10 }) -) - -const results = await Promise.all(promises) -// ✅ Only 1 rebuild triggered (mutex) -// ✅ All 100 queries return correct results -// ✅ Total time: ~60ms (not 6000ms!) -``` - -**Use Cases for Lazy Loading:** -- **Serverless/Edge**: Minimize cold start time (0-10ms init) -- **Development**: Faster restarts during development -- **Large datasets**: Defer index loading until needed -- **Read-heavy workloads**: Writes don't wait for index rebuild + ## Zero Configuration Required @@ -384,10 +350,6 @@ Brainy is designed to be **smart enough to tune itself dynamically**. No configu // That's it. Brainy handles everything. const brain = new Brainy() await brain.init() - -// Or with lazy loading for serverless -const brain = new Brainy({ disableAutoRebuild: true }) -await brain.init() // Instant (0-10ms) ``` ### Automatic Self-Tuning @@ -395,7 +357,6 @@ await brain.init() // Instant (0-10ms) - **Metadata Index**: Auto-builds sorted indices for range queries on first use - **Graph Index**: Auto-flushes every 30 seconds - **Default Tuning**: Research-based vector index defaults -- **Lazy Loading**: Indices built only when needed - **Cache Management**: LRU caches with TTL ### Intelligent Defaults diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index d9a4d3e7..3078c239 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -10,7 +10,7 @@ next: - guides/storage-adapters --- -# Plugin Development Guide +# Plugin System Brainy has a plugin system that allows third-party packages to replace internal subsystems with custom implementations. This is how `@soulcraft/cor` provides optional native acceleration, and it's the same system available to any developer. @@ -200,15 +200,30 @@ members so a warm reopen never pays a redundant rebuild-from-canonical: - **`init?(): Promise`** — eager cold-load. Brainy awaits it once during `brain.init()`, after the metadata provider's `init()` (the id-mapper hydrates first) and **before the rebuild gate**. -- **`isReady?(): boolean`** — honest durability signal. `true` ⇔ the persisted index is - loaded (or cheaply demand-loadable) and consistent with what was last persisted. When - exposed, the rebuild gate defers to this signal **instead of** the `size() === 0` / - `totalEntries === 0` heuristics — a disk-native index may report 0 resident entries - while fully durable. Never return `true` if the durable state failed to load: the - signal is honest in both directions, and a not-ready provider gets its rebuild even - when `size() > 0`. +- **`healthReport?(): HealthReport`** — the PREFERRED signal (10.4+). A named, + synchronous, O(1) verdict derived from the provider's own exact ledgers — never a + sample, never I/O, must never throw for a well-formed provider. Brainy's read gate + (`assessProviderHealth()`) reads this INSTEAD of `isReady()` / size heuristics when + present: `serving: false` refuses the read with a typed `*NotReadyError` rather than + triggering a rebuild — a read never starts a store walk. `healthy` marks every + *verified* invariant holding; a family named in `unledgered` counts as neither + healthy nor broken. See `HealthReport` / `LedgerInvariantResult` / + `InvariantSource` in `src/plugin.ts`, and + [Index Health](concepts/index-health.md) for the consumer-facing story. +- **`isReady?(): boolean`** — honest durability signal, the fallback when + `healthReport()` is absent. `true` ⇔ the persisted index is loaded (or cheaply + demand-loadable) and consistent with what was last persisted. When exposed, the + gate defers to this signal **instead of** the `size() === 0` / `totalEntries === 0` + heuristics — a disk-native index may report 0 resident entries while fully durable. + Never return `true` if the durable state failed to load: the signal is honest in + both directions, and a not-ready provider gets its rebuild even when `size() > 0`. - **`isMigrating?(): boolean`** — while `true`, the provider owns its index (background migration); brainy skips its rebuild entirely. +- **`validateInvariants?(): Promise`** — the async DEEP + diagnostic (full scans allowed), distinct from the bounded, sync `healthReport()`. + Must never throw — a failure is `healthy: false` data, not an exception; a provider + that throws anyway is read as a loud, unverified failure (never as "healthy") by + every caller, never silently retried into a rebuild. Providers that implement none of these keep the size/count heuristics — correct for engines whose `rebuild()` *is* their load path (like brainy's built-in JS vector index). diff --git a/docs/api/README.md b/docs/api/README.md index fb9cc920..82f48c91 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -1451,6 +1451,34 @@ const count = await brain.getVerbCount() --- +### The canonical count ledger (`StorageAdapter.getCanonicalCounts()`) + +An OPTIONAL method on the `StorageAdapter` interface (implemented by both +built-in adapters), not a method on `Brainy` itself — relevant if you're +writing a custom storage adapter or composing a provider's own +`healthReport()`. O(1), no I/O. Per family (`nouns`/`verbs`): + +```typescript +interface CanonicalCounts { + nouns: { counted: number; all: number } + verbs: { counted: number; all: number } + suspect: boolean +} +``` + +- `counted` mirrors `getNounCount()` / `getVerbCount()` (public + internal tiers). +- `all` is the ALL-visibility scalar — every tier, including system/internal + records — the denominator a derived index's own coverage math is measured + against. +- `suspect` is `true` when an unprovable delete has left `all` unverified since + the last recount; `brain.repairIndex()` clears it with a real canonical walk. + +Adapters without the ledger omit the method; treat absence as "no +denominator," never as zero. See +**[Index Health](../concepts/index-health.md)** for the full story. + +--- + ### Subtype & facet APIs Full guide: **[Subtypes & Facets](../guides/subtypes-and-facets.md)**. @@ -1852,6 +1880,81 @@ await brain.repairIndex({ rebuild: ['graph'] }) await brain.repairIndex({ rebuild: 'all' }) ``` +**`RepairReport`:** +- `families: RepairFamilyReport[]` — one row per family checked +- `healedTotal: number` — items healed across every family +- `durationMs: number` + +**`RepairFamilyReport`** (one row): +- `family: string` — e.g. `'orphaned-containers'`, `'count-rollups'`, + `'vfs-containment'`, `'metadata-corruption'`, `'provider:metadata'`, + `'provider:graph'`, `'provider:vector'` +- `checked: boolean` — was this family actually examined (`false` ⇒ see `skipped`) +- `healed: number` — items re-posted/corrected in place (the incremental heal count) +- `missing?: { count: number; sample: string[] }` — exact count plus a capped id + sample when the check can name what diverged (never the full list) +- `rebuilt?: boolean` — a full generational rebuild ran (vs. an incremental heal) +- `detail?: string` / `reason?: string` — narration +- `skipped?: string` — why the family wasn't checked + +Full walkthrough — what each family checks, degraded-but-serving vs. not-ready, +and what `suspect` counts mean — in +**[Index Health](../concepts/index-health.md)**. + +--- + +### Index readiness: typed errors, `healthReport()`, `disableAutoRebuild` + +Every derived-index provider (vector, graph, metadata) may expose a named, +synchronous, O(1) `healthReport()` composed from its own exact ledgers — the +signal Brainy's read gate trusts over sampling or size heuristics. `init()` +brings every provider to serving before it returns; there is no first-query +lazy-rebuild path. A read that reaches a provider whose health report says it +isn't serving throws instead of rebuilding mid-query: + +| Error | Thrown by | Meaning | +|---|---|---| +| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | Graph adjacency isn't serving | +| `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving | +| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving | + +All three are exported from `@soulcraft/brainy`. Catch them to distinguish +"index not ready" from a genuine empty result: + +```typescript +import { MetadataIndexNotReadyError } from '@soulcraft/brainy' + +try { + const rows = await brain.find({ where: { status: 'active' } }) +} catch (err) { + if (err instanceof MetadataIndexNotReadyError) { + // reconcile: await brain.repairIndex(), then retry + } else { + throw err + } +} +``` + +**`disableAutoRebuild`** no longer defers index construction to the first +query. A needed rebuild always runs at `open()`, regardless of this flag or +dataset size; the flag has no effect on *when* a rebuild runs. Full manual +control lives in `repairIndex({ rebuild: [...] })`, above. + +### `validateIndexConsistency()` → `Promise<...>` + +The deep, async diagnostic counterpart to `healthReport()` — safe to run on a +live brain, but does more work (a provider's `validateInvariants()` may run a +full scan, not just read a ledger). Aggregates the JS metadata index's own +consistency check with every derived-index provider's invariant report. + +```typescript +const validation = await brain.validateIndexConsistency() +if (!validation.healthy) { + console.log(validation.recommendation) // what to run, e.g. repairIndex() + console.log(validation.providers) // each provider's own invariant report, when exposed +} +``` + --- ## Lifecycle diff --git a/docs/architecture/index-architecture.md b/docs/architecture/index-architecture.md index 8b3dc540..6a754b56 100644 --- a/docs/architecture/index-architecture.md +++ b/docs/architecture/index-architecture.md @@ -723,6 +723,14 @@ async stats(): Promise { ### 5. Index Rebuilding (Lazy Loading Support) +> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is +> RETIRED.** `disableAutoRebuild` no longer defers index construction to a +> first query; `brain.init()` now runs every needed rebuild to completion +> before it returns, unconditionally, and a read against a not-serving +> provider throws a typed `*NotReadyError` instead of rebuilding mid-query. +> See `docs/concepts/index-health.md` for the current contract. Left below +> as historical background on the rebuild mechanics. + **Two modes of index loading:** #### Mode 1: Auto-Rebuild on init() (default) diff --git a/docs/architecture/initialization-and-rebuild.md b/docs/architecture/initialization-and-rebuild.md index a1645744..e19bdd9f 100644 --- a/docs/architecture/initialization-and-rebuild.md +++ b/docs/architecture/initialization-and-rebuild.md @@ -1,5 +1,15 @@ # Initialization and Rebuild Processes +> **Stale as of 10.4 — "Mode 2: Lazy Loading on First Query" below is RETIRED.** +> `disableAutoRebuild` no longer defers index construction to a first query; +> `brain.init()` now runs every needed rebuild to completion before it +> returns, unconditionally. A read against a not-serving provider throws a +> typed `*NotReadyError` instead of rebuilding mid-query. See +> `docs/concepts/index-health.md` for the current contract; this document's +> line-number references to `src/brainy.ts` also predate the file's current +> size and are unreliable. Left as historical background on the rebuild +> mechanics, not as a current API description. + This document explains how Brainy's four indexes (MetadataIndex, vector index, GraphAdjacencyIndex, DeletedItemsIndex) initialize and rebuild from persisted storage. ## Core Principle: All Indexes Are Disk-Based diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md new file mode 100644 index 00000000..18bf7010 --- /dev/null +++ b/docs/concepts/index-health.md @@ -0,0 +1,204 @@ +--- +title: Index Health +slug: concepts/index-health +public: true +category: concepts +template: concept +order: 8 +description: How Brainy knows whether a derived index can be trusted — exact accounting instead of sampling, the named health report, degraded-but-serving vs. not-ready, and what repairIndex() checks, heals, and rebuilds. +next: + - concepts/generation-fact-log + - guides/inspection +--- + +# Index Health + +Brainy keeps one **canonical** copy of every entity and relationship, and three +**derived** indexes built from it — vector, metadata, and graph — so `find()` can +answer semantically, by filter, and by traversal without re-deriving the answer from +scratch on every query. A derived index is a cache with a serving structure: it can +be present but stale, present but only partially loaded, or fully out of sync with +canonical after a crash. This page is about how Brainy decides whether to trust one, +what it does when it can't, and how you reconcile the two. + +## Exact accounting instead of sampling + +Older health checks worked by inference: does `size()` return something greater +than zero, does a spot-check on one known item come back correct. Both are proxies. +A cold index can report a nonzero count while its actual serving structure never +loaded, and a spot-check only proves the one item it happened to ask about. + +Every derived-index provider may now expose a named, synchronous, O(1) +`healthReport()` — composed from the provider's own **exact ledgers** (real counters +it already maintains on the write path), never a sample or a walk. This is the one +signal Brainy's read gate consults. A provider that doesn't yet expose one falls +back to an honest `isReady()` boolean, and finally to a size heuristic for engines +with neither — but wherever a `healthReport()` exists, it wins. + +Underneath, storage itself keeps an analogous **canonical count ledger**: a +`counted` scalar (the user-facing total — what `getNounCount()` / `getVerbCount()` +return) and an `all` scalar (every tier, including internal records a derived +index's own coverage math needs to compare against). This is the real denominator +a provider's `healthReport()` measures itself by, rather than a total that can only +ever ratchet upward. See [What `suspect` counts mean](#what-suspect-counts-mean) +below for the one case that ledger can't stay exact through on its own. + +## The named report + +A `HealthReport` carries, per provider (`'vector'` / `'graph'` / `'metadata'`): + +- **`healthy`** — `true` iff every *verified* invariant holds. An invariant whose + family has no ledger yet is `unledgered`, never counted either way — unknown, + not passing. +- **`serving`** — can this provider answer a query right now. A failing invariant + graded `heal: 'repair'` or `heal: 'none'` still leaves `serving: true` — this is + **degraded-but-serving**: something is off (say, a stale rollup on an + `employee` record's relationship count) but reads keep working. Only a failure + graded `heal: 'rebuild'` flips `serving` to `false` — **not-ready** — because the + provider itself is telling you its serving structure cannot answer correctly. +- **`invariants`** — each checked condition, with its provenance + (`source: 'ledger'` — an exact count; `'deep'` — a full scan, diagnostic-only; + `'unledgered'` — not yet tracked) and, for a failing one, an exact `missing` + count plus a capped sample of the affected ids — a verdict, never a dump. +- **`generation`** — bumps on every ledger mutation and rebuild, so a caller can + cache a verdict per generation instead of re-deriving it. + +The distinction that matters day to day: `healthy: false` can be entirely benign — +a maintenance window, a divergence `repairIndex()` will clean up on its own +schedule. `serving: false` is not benign. It means this provider is refusing to +answer, on its own word, right now. + +## Reads refuse — they never rebuild + +A query that reaches a not-serving provider does not trigger a rebuild from inside +the read. Brainy retired that path deliberately: a rebuild kicked off by an ordinary +`find({ where: { status: 'active' } })` call is a dark, unpredictable cost hiding +behind a request that looks like a cheap read. Instead, the read throws a typed, +catchable error naming the reason: + +| Error | Thrown when | Meaning | +|---|---|---| +| `GraphIndexNotReadyError` | `find({ connected })`, `neighbors()`, `related()` | The graph adjacency index isn't serving — traversal would otherwise return `[]` indistinguishable from "no relationships" | +| `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" | +| `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" | + +All three are exported from `@soulcraft/brainy`. Catch them where your application +needs to distinguish "this index isn't ready yet" from "there's genuinely nothing +here" — a health dashboard, a retry policy, an operator alert. The fix is always +the same: reconcile the index, either by reopening the brain (which brings every +provider to serving before `init()` returns — see the next section) or by calling +`repairIndex()` explicitly. + +```typescript +try { + const active = await brain.find({ where: { status: 'active' } }) +} catch (err) { + if (err instanceof MetadataIndexNotReadyError) { + // not a "no results" — the index itself refused; alert or retry after repair + } else { + throw err + } +} +``` + +### Rebuilds happen at open, not on first query + +`brain.init()` runs every needed rebuild to completion **before it returns**, +unconditionally, regardless of dataset size. There is no lazy, first-query +rebuild path anymore — a brain either finishes opening healthy, or it fails +open loudly. `disableAutoRebuild: true` no longer defers index construction to +the first query: it has no effect on *when* a needed rebuild runs. Full manual +control over rebuilds is `repairIndex({ rebuild: [...] })` (below), not this flag. + +## `repairIndex()` — checking and healing + +Bare `repairIndex()` is **report-driven**: it only heals what its own checks say +actually needs it, and it always returns a full per-family receipt. + +```typescript +const report = await brain.repairIndex() +report.healedTotal // total items healed across every family +report.durationMs +report.families // one row per family checked +``` + +Each `RepairFamilyReport` row names what happened: + +- **`checked`** — was this family actually examined (`false` means skipped — + see `skipped` for why). +- **`healed`** — items re-posted or corrected in place. +- **`missing`** — when the check can name what diverged: an exact `count` plus a + capped `sample` of ids. +- **`rebuilt`** — a full generational rebuild ran (as opposed to an incremental + heal). +- **`detail`** / **`reason`** / **`skipped`** — the receipt's narration; a row is + always either checked or explains why it wasn't. Nothing is silent. + +On every call, bare `repairIndex()`: + +1. Prunes orphaned canonical containers left by a partial delete. +2. Recomputes the count rollups from one canonical walk (unconditional — this is + also what clears a `suspect` ledger; see below). +3. Reconciles VFS containment edges, if the VFS is initialized. +4. Runs the metadata index's own corruption detection pass. +5. Consults each of the three derived-index providers' own health check and + rebuilds only a family whose failing invariant actually asks for it + (`heal: 'rebuild'`) — never a provider that reports `healthy` or a lesser + grade. + +### The explicit rebuild door + +`options.rebuild` skips the health check and rebuilds one or more families +**unconditionally** — the operator override for when you have independent reason +to distrust a family regardless of what it self-reports (a suspicious deploy, a +storage-layer incident, a support ticket that doesn't match what the health report +says): + +```typescript +// Force the graph adjacency to rebuild from canonical, no invariant consulted +await brain.repairIndex({ rebuild: ['graph'] }) + +// Force all three derived indexes +await brain.repairIndex({ rebuild: 'all' }) +``` + +A family named this way is recorded with `rebuilt: true` and +`reason: 'explicit rebuild requested'`, and is skipped by the normal +health-driven pass in the same call — it was already rebuilt unconditionally. + +Reach for the explicit door when you need certainty regardless of self-report; +reach for bare `repairIndex()` for routine maintenance and after any incident +where you're not sure which family (if any) needs it. + +## What `suspect` counts mean + +Storage's canonical count ledger increments the ALL-visibility total on every new +record and decrements it on every *proven* delete — one where the record was read, +or the caller supplied its prior image. A delete that cannot prove what it removed +existed doesn't guess: it flags the ledger `suspect` (an operator-visible +`console.warn`, narrated once per session, not once per delete) rather than risk +decrementing a total that was never incremented for that record in the first +place. This is intentionally rare — it's a defensive fallback for callers on an +unusual removal path, not a per-delete cost. + +`suspect` is not directly exposed on any `Brainy` method today — it lives on the +`StorageAdapter`'s optional `getCanonicalCounts()`, primarily consulted by +`repairIndex()`'s recount step and by custom storage adapters composing their own +`healthReport()`. What matters for an application: a `suspect` ledger is not +incorrect, just *unverified since the last recount* — and `repairIndex()`'s +unconditional count-rollup step (step 2, above) recomputes the ALL scalars from a +real canonical walk on every call, clearing the flag with proof either way. + +## Practical guidance + +- **On a normal restart**, do nothing — `init()` brings every provider to + serving before it returns, or fails loudly. +- **On a `*NotReadyError`** from a live read, reconcile with `repairIndex()` + (report-driven is almost always sufficient) and retry. +- **After an incident** where you distrust a specific family regardless of what + it reports healthy — a storage-layer fault, a suspicious restore — use the + explicit door: `repairIndex({ rebuild: ['metadata' | 'graph' | 'vector'] })`. +- **To audit before trusting a report**, `brain.auditGraph()` walks every stored + relationship and proves (or disproves) that reads return canonical truth, + independent of what any provider self-reports — see + [Inspecting a Live Brainy](../guides/inspection.md). diff --git a/tests/lifecycle/README.md b/tests/lifecycle/README.md index e6456f19..ebe0e61d 100644 --- a/tests/lifecycle/README.md +++ b/tests/lifecycle/README.md @@ -14,3 +14,9 @@ from the message alone. `biography.test.ts` is split into two `it` blocks (Ch1-3, then Ch4-6) purely for reporting; it is still ONE fixed-order story. Chapters must never be reordered, skipped, or made conditional, and a failing chapter's assertion must never be weakened to force green. + +Lab notes (hard-won, keep): +- `git reset --hard` does NOT remove untracked files — a "clean" tree can still + carry stray test stores; use `git clean -fd tests/lifecycle-tmp` equivalents. +- `silent: true` patches `console` process-wide — never assert narration through + `console` spies in this lane; the engine's always-on channel is `prodLog`. From 96624f408cadcb824cdd2dabcf40aceb9a78fd8b Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 10:09:45 -0700 Subject: [PATCH 011/150] feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production restart storm measured 90,017ms for a single brain init vs 1,117ms quiet (~80x contention multiplier), traced to performInit() eagerly awaiting the process-global WASM embedding engine before the VFS root even existed. Every writer's open() queued on the one throttled model compile (90-140s on throttled CPUs). - VirtualFileSystem.doInitializeRoot() no longer embeds '/'. The root is system-tier plumbing nothing ever searches; when the default WASM engine is active it now gets an explicit all-zero placeholder vector (cosineDistance returns max distance for a zero vector, so it never ranks ahead of real content). deferEmbedding was considered and rejected: its landing path kicks the embed worker synchronously right after commit, which would still force the cold compile within milliseconds — just off the awaited path, not avoided. A registered native 'embeddings' provider (no cold-start cost, possibly a different dimension) still embeds the root for real, via the new Brainy.usesDefaultWasmEmbedder() seam. - performInit()'s eager-embedding step now only STARTS the WASM engine warm in the background instead of awaiting it inline. embed()/embeddingManager already serialize concurrent callers on one shared init promise, so the first real embed() converges correctly either way; a failed warm narrates loudly instead of surfacing as a silent latency spike or an unhandled rejection. eagerEmbeddings: false still means no warm at all. - FileSystemStorage.init() batches its ~8 independent bootstrap mkdirs (each creates its own full subtree via recursive:true, so none depend on the others existing) into one Promise.all. The restore-completion step and initializeCounts() stay strictly sequential — they have real order dependencies on rootDir and systemDir respectively. - performInit() now times five phases (storage init / generation-store open+fold / index init+gate / VFS bootstrap / embedding-warm-started) and logs one warning with the per-phase breakdown when total open exceeds 2000ms; silent otherwise. --- src/brainy.ts | 146 ++++++++++++++-- src/storage/adapters/fileSystemStorage.ts | 74 ++++---- src/types/brainy.types.ts | 33 ++-- src/vfs/VirtualFileSystem.ts | 47 ++++- tests/unit/brainy/open-path.test.ts | 199 ++++++++++++++++++++++ 5 files changed, 446 insertions(+), 53 deletions(-) create mode 100644 tests/unit/brainy/open-path.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index c8912e26..9cd7dcb4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -742,6 +742,15 @@ export class Brainy implements BrainyInterface { private _pendingEmbedIds = new Set() private _embedWorkerFlight: Promise | null = null + // OPEN-PATH FIX: the background embedding-engine warm kicked off (never + // awaited) by `performInit()` when `eagerEmbeddings` resolves true. Stored + // for observability only — `embed()`/`embeddingManager.embed()` already + // await the engine's OWN singleton init promise internally, so nothing + // needs to explicitly await this field for correctness. Never rejects on + // its own: a `.catch` narrates the failure and swallows it so a failed + // warm never surfaces as an unhandled rejection. + private _embeddingWarmPromise: Promise | null = null + /** The stored log-authority switch, read once at open (default: tree). */ private _logAuthority: LogAuthorityRecord = { authority: 'tree' } // A failed walk latches its error: retries within the cooldown rethrow it @@ -1079,6 +1088,23 @@ export class Brainy implements BrainyInterface { configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging } + // OPEN-PATH NARRATION: lightweight phase timing across the five + // named stretches of init — storage init / generation-store open+fold / + // index init+gate / VFS bootstrap / embedding-warm-started. Each + // `markPhase()` call records elapsed ms SINCE THE PREVIOUS checkpoint, + // so the buckets always sum to the pre-integration/warmOnOpen total. + // Silent under 2s; one `prodLog.warn` line naming every phase's ms + // above it, so the operator's next restart storm names its own slow + // phase instead of re-deriving it from a stack of raw timestamps. + const initStart = Date.now() + let lastPhaseCheckpoint = initStart + const phaseTimingsMs: Record = {} + const markPhase = (name: string): void => { + const now = Date.now() + phaseTimingsMs[name] = now - lastPhaseCheckpoint + lastPhaseCheckpoint = now + } + try { // Auto-detect and activate plugins BEFORE storage setup // so plugin-provided storage factories (e.g., filesystem override from cor) are available @@ -1150,6 +1176,12 @@ export class Brainy implements BrainyInterface { } } + // PHASE 1 of 5 — "storage init": plugin/legacy-layout bootstrap, + // storage adapter construction+init, the OS-limit check, and the + // writer-lock claim, all folded into one bucket (everything above this + // line since performInit started). + markPhase('storage-init') + // 8.0 generational MVCC: open the record layer BEFORE any index is // created or loaded. Crash recovery may rewrite canonical entity files // (restoring before-images of an uncommitted transaction), and every @@ -1224,6 +1256,12 @@ export class Brainy implements BrainyInterface { await this.createMigrationBackupIfNeeded() } + // PHASE 2 of 5 — "generation-store open+fold": GenerationStore + // construction+open (crash-recovery replay/rollback fold), the + // derived-family registration, the fact-scan seam, the entity-tree + // stamp check, the brain-format handshake, and the pre-upgrade backup. + markPhase('generation-store-open-fold') + // Provider: embeddings (reassign embedder if plugin provides one) const embeddingProvider = this.pluginRegistry.getProvider('embeddings') if (embeddingProvider) { @@ -1461,6 +1499,14 @@ export class Brainy implements BrainyInterface { // Check for pending data migrations await this.checkMigrations() + // PHASE 3 of 5 — "index init+gate": provider wiring (embeddings, + // cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph + // index construction, the eager cold-load, id-resolver + connections- + // codec wiring, crash-recovery index rebuild, the replay-gap check, + // legacy VFS blob adoption, blob-history backfill, and the + // rebuildIndexesIfNeeded() gate + migration check. + markPhase('index-init-gate') + // Register shutdown hooks for graceful count flushing (once globally) if (!Brainy.shutdownHooksRegisteredGlobally) { this.registerShutdownHooks() @@ -1612,15 +1658,40 @@ export class Brainy implements BrainyInterface { } } - // Eager embedding initialization. + // PHASE 4 of 5 — "VFS bootstrap": shutdown-hook registration, blob + // storage init, the provider-summary log, flipping `initialized`, + // the migration-lock wait, VFS construction+init, flipping generation + // stamping active, the log-authority adopt/oracle check, and + // pending-embed crash recovery. + markPhase('vfs-bootstrap') + + // Eager embedding initialization — BACKGROUND WARM (open-path fix). // - // Adaptive default (8.0): the WASM embedding engine eagerly initializes + // Adaptive default (8.0): the WASM embedding engine eagerly WARMS // during init() WHENEVER it is the active embedder — i.e. no native // 'embeddings' provider has taken over — and the instance is a writer // (not reader-mode) outside of unit tests. The WASM module (≈93MB with - // the embedded model) takes 90-140s to compile on throttled CPUs; paying - // that during boot rather than on the first embed()-driven call is the - // right default for the overwhelmingly common single-process server. + // the embedded model) takes 90-140s to compile on throttled CPUs. + // + // Historically this AWAITED `embeddingManager.init()` INLINE, so every + // writer's open() blocked on the compile — N concurrent opens all + // queued on the ONE process-global singleton (an ~80x contention + // multiplier measured in a production restart storm: 90,017ms busy vs + // 1,117ms quiet). The engine only needs to be ready before the FIRST + // REAL embed() call, not before init() returns, so this now only + // STARTS the warm and moves on — init() never waits for it. + // + // No double-await needed for correctness: `this.embed()` (~line 15420) + // delegates to `this.embedder`, which for the default engine is + // `embeddingManager.getEmbeddingFunction()` → `embeddingManager.embed()` + // (src/embeddings/EmbeddingManager.ts). That method calls `await + // this.init()` FIRST, and `init()` itself serializes every concurrent + // caller onto ONE shared `globalInitPromise` — so the first real + // embed() automatically waits for whichever finishes first: this + // background warm (if still running) or a fresh init() (if the warm + // hasn't reached this code yet, e.g. `eagerEmbeddings: false`). + // Verified by reading both call sites; `_embeddingWarmPromise` below + // is stored for observability only, never re-awaited by embed(). // // Skipped automatically when: // - a native 'embeddings' provider is registered (it owns embeddings; @@ -1628,8 +1699,8 @@ export class Brainy implements BrainyInterface { // - reader-mode (readers don't embed — they query existing vectors), // - unit-test mode (tests must stay fast and use the mock embedder). // - // `eagerEmbeddings: false` is the explicit override to force lazy init - // (first-embed) even when this instance is the active embedder. + // `eagerEmbeddings: false` keeps meaning "no warm at all" — fully lazy, + // the first embed() call pays the full cost inline, same as before. const isUnitTestMode = isDeterministicEmbedMode() const eager = this.config.eagerEmbeddings ?? true if ( @@ -1638,9 +1709,45 @@ export class Brainy implements BrainyInterface { this.config.mode !== 'reader' && !isUnitTestMode ) { - console.log('Eager embedding initialization enabled...') - await embeddingManager.init() - console.log('Embedding engine ready') + const warmStart = Date.now() + console.log('Background embedding-engine warm started (init() does not wait for it)...') + this._embeddingWarmPromise = embeddingManager + .init() + .then(() => { + prodLog.info( + `[Brainy] background embedding-engine warm complete in ${Date.now() - warmStart}ms` + ) + }) + .catch((err) => { + // Loud, never silent: a warm that fails to compile must be + // heard NOW, not discovered as a mystery latency spike on + // whichever request happens to trigger the first real embed(). + // That first embed() call still retries init() itself (the + // singleton promise contract above) and surfaces its own typed + // error to its caller — this is the immediate, background echo. + prodLog.warn( + `[Brainy] background embedding-engine warm FAILED: ` + + `${(err as Error).message} — the first embed() call will retry ` + + `initialization and surface the error there` + ) + }) + } + + // PHASE 5 of 5 — "embedding-warm-started": just the synchronous cost + // of kicking off the background warm above (the warm's own compile + // time is NOT included — that's the whole point of backgrounding it). + markPhase('embedding-warm-started') + { + const totalOpenMs = Date.now() - initStart + if (totalOpenMs > 2000) { + const phaseList = Object.entries(phaseTimingsMs) + .map(([name, ms]) => `${name}=${ms}ms`) + .join(', ') + prodLog.warn( + `[Brainy] slow open: ${totalOpenMs}ms total (${phaseList}) — see the ` + + `phase breakdown above to find which one to investigate first` + ) + } } // Integration Hub initialization @@ -15695,6 +15802,25 @@ export class Brainy implements BrainyInterface { return embeddingManager.isInitialized() } + /** + * Whether the process-global WASM embedding engine (all-MiniLM-L6-v2, + * fixed 384-dim output, ≈93MB with the bundled model, 90-140s cold compile + * on throttled CPUs) is this instance's active embedder — `false` when a + * plugin has replaced it via the `'embeddings'` provider key. A native + * provider has no such cold-start cost and may use a different output + * dimension, so it is never worth avoiding. + * + * Used by init-path bootstrap writes (the VFS root — see + * `VirtualFileSystem.doInitializeRoot()`) to decide whether embedding a + * value during `init()` risks paying the WASM engine's cold compile. + * + * @returns true when the default WASM engine is active (no native + * `'embeddings'` provider registered). + */ + usesDefaultWasmEmbedder(): boolean { + return !this.pluginRegistry.hasProvider('embeddings') + } + /** * Setup embedder */ diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 6d2d9b3c..965e97b9 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -239,38 +239,54 @@ export class FileSystemStorage extends BaseStorage { // Finish any restore interrupted by a crash (resume the staged swap, or // discard an uncommitted staging area) BEFORE counts/derived state load, - // so the rest of startup sees the completed store. + // so the rest of startup sees the completed store. ORDER-DEPENDENT: + // `swapStagedRestoreIn()` reads `fs.readdir(rootDir)` and then + // removes/renames rootDir's own TOP-LEVEL entries to place the staged + // copy — racing that against the directory-creation batch below (which + // also touches rootDir's children) could see a half-created directory + // mid-swap or a mkdir racing a concurrent rm/rename on the same path. + // Stays strictly sequential, never folded into the OPEN-PATH batch. await this.completeInterruptedRestore() - // Create the nouns directory if it doesn't exist - await this.ensureDirectoryExists(this.nounsDir) + // OPEN-PATH FIX: the remaining bootstrap directories are mutually + // independent — each is its own subtree under rootDir, and + // `fs.mkdir(dir, { recursive: true })` creates every intermediate + // segment of ITS OWN path in one call, so it never depends on any + // sibling here existing first. Nothing between here and + // `initializeCounts()` reads any of them, so batching collapses what + // was up to 8 sequential mkdir round-trips (each a real syscall+await) + // into one wave — this is what serialized an N-writer restart storm on + // filesystem I/O it never structurally needed. `initializeCounts()` + // right after DOES depend on `systemDir` (which the batch creates), so + // it stays outside, awaited only once every directory has landed. + await Promise.all([ + // Create the nouns directory if it doesn't exist + this.ensureDirectoryExists(this.nounsDir), + // Create the verbs directory if it doesn't exist + this.ensureDirectoryExists(this.verbsDir), + // Create the metadata directory if it doesn't exist + this.ensureDirectoryExists(this.metadataDir), + // Create the noun metadata directory if it doesn't exist + this.ensureDirectoryExists(this.nounMetadataDir), + // Create the verb metadata directory if it doesn't exist + this.ensureDirectoryExists(this.verbMetadataDir), + // Create both directories for backward compatibility + this.ensureDirectoryExists(this.systemDir), + // Only create legacy directory if it exists (don't create new legacy + // dirs) — a read-then-maybe-write, but on its own subtree, so it's + // still independent of every other entry in this batch. + (async () => { + if (await this.directoryExists(this.indexDir)) { + await this.ensureDirectoryExists(this.indexDir) + } + })(), + // Create the locks directory if it doesn't exist + this.ensureDirectoryExists(this.lockDir), + // Create the binary blobs directory if it doesn't exist + this.ensureDirectoryExists(this.blobsDir) + ]) - // Create the verbs directory if it doesn't exist - await this.ensureDirectoryExists(this.verbsDir) - - // Create the metadata directory if it doesn't exist - await this.ensureDirectoryExists(this.metadataDir) - - // Create the noun metadata directory if it doesn't exist - await this.ensureDirectoryExists(this.nounMetadataDir) - - // Create the verb metadata directory if it doesn't exist - await this.ensureDirectoryExists(this.verbMetadataDir) - - // Create both directories for backward compatibility - await this.ensureDirectoryExists(this.systemDir) - // Only create legacy directory if it exists (don't create new legacy dirs) - if (await this.directoryExists(this.indexDir)) { - await this.ensureDirectoryExists(this.indexDir) - } - - // Create the locks directory if it doesn't exist - await this.ensureDirectoryExists(this.lockDir) - - // Create the binary blobs directory if it doesn't exist - await this.ensureDirectoryExists(this.blobsDir) - - // Initialize count management + // Initialize count management — depends on systemDir, created above. this.countsFilePath = path.join(this.systemDir, 'counts.json') await this.initializeCounts() diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 38bc4aa6..63356828 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1983,25 +1983,32 @@ export interface BrainyConfig { reservedQueryMemory?: number // Memory reserved for queries in bytes (e.g., 1073741824 = 1GB) /** - * Controls when the WASM embedding engine is initialized. + * Controls whether `init()` starts a BACKGROUND warm of the WASM embedding + * engine. * - * **Adaptive default (8.0):** when omitted, the engine eagerly initializes - * during `init()` whenever the WASM embedder is the *active* one — i.e. no - * native `'embeddings'` provider is registered — and this instance is a - * writer (not `mode: 'reader'`) running outside unit tests. The WASM module - * (≈93MB with the embedded model) takes 90-140s to compile on throttled - * CPUs, so paying that during boot rather than on the first `embed()`-driven - * call is the right default for a single-process server. + * **Adaptive default (8.0, background since the open-path fix):** when + * omitted, `init()` STARTS a background warm of the engine whenever the + * WASM embedder is the *active* one — i.e. no native `'embeddings'` + * provider is registered — and this instance is a writer (not + * `mode: 'reader'`) running outside unit tests. The WASM module (≈93MB with + * the embedded model) takes 90-140s to compile on throttled CPUs — but + * `init()` never awaits that compile. It only starts it, so N concurrent + * opens no longer serialize on the one process-global engine singleton. + * The first `embed()` call then waits for whichever finishes first: the + * background warm (if still running) or its own fresh init (if the warm + * never started, e.g. `eagerEmbeddings: false`) — both paths converge on + * the SAME shared promise inside the engine singleton, so the vector is + * always correct; only the timing of who pays the wait differs. * * The adaptive path skips itself automatically when a native embeddings * provider owns embeddings, in reader-mode (readers query existing vectors * and never embed), and in unit-test mode (kept fast via the mock embedder). * - * - `true` — force eager init during `init()` (the adaptive default already - * does this for the active-embedder writer case; set it explicitly to be - * unambiguous). - * - `false` — explicit override to force lazy init (first `embed()` call) - * even when this instance is the active embedder. + * - `true` — force the background warm to start during `init()` (the + * adaptive default already does this for the active-embedder writer + * case; set it explicitly to be unambiguous). + * - `false` — no warm at all. Fully lazy: the first `embed()` call pays the + * full cold-compile cost inline, on whichever request triggers it. */ eagerEmbeddings?: boolean diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index ed272109..0a000396 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -89,6 +89,16 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Uses deterministic UUID format for storage compatibility private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + // OPEN-PATH FIX: the dimension of the placeholder vector given to the VFS + // root when it is first created (see `doInitializeRoot`). Mirrors the + // built-in WASM embedding engine's fixed, hardcoded output size + // (all-MiniLM-L6-v2 — see src/embeddings/candle-wasm/src/lib.rs + // HIDDEN_SIZE and src/embeddings/EmbeddingManager.ts). Deliberately NOT + // derived from `brain.dimensions` — this constant only applies to the + // default-embedder branch, where the true output dimension is this fixed + // value by construction, never a moving target. + private static readonly VFS_ROOT_VECTOR_DIMENSIONS = 384 + /** * Construct a VFS bound to a Brainy instance. * @@ -260,6 +270,40 @@ export class VirtualFileSystem implements IVirtualFileSystem { try { console.log('VFS: Creating root directory (fixed ID: 00000000-0000-0000-0000-000000000000)') + // OPEN-PATH FIX: the VFS root is Brainy's own system-tier plumbing — it + // is hidden from find()/getNounCount()/stats() by default and nothing + // ever runs a semantic search against it — so it needs no REAL + // embedding. Historically this add() always called embed('/'), which + // meant every writer's FIRST-EVER open forced the process-global WASM + // engine to cold-compile its model (measured 90-140s on throttled + // CPUs) before the brain could even finish init(). This branch only + // runs once per store (the root already exists — with a real vector — + // in every previously-opened production store; reopening never re-adds + // it), so the fix applies only to brand-new stores. + // + // Chosen fix: an explicit all-zero vector, not `deferEmbedding: true`. + // `deferEmbedding` looked attractive (ack now, embed later) but its + // landing path (`kickEmbedWorker()`, called synchronously right after + // commit — see brainy.ts add()/update()) would still force the WASM + // engine to cold-compile within milliseconds of open, just off the + // awaited path instead of never paying it at all — worse than the + // explicit-vector path, which never touches the engine for this row. + // An all-zero vector is safe: `cosineDistance` (src/utils/distance.ts) + // explicitly returns the MAXIMUM distance whenever either operand's + // norm is zero, so the root can never rank ahead of real content in a + // similarity search, and HNSW indexes it like any other vector. + // + // Only the default WASM engine gets this treatment — its output + // dimension (384) is fixed and hardcoded, so the placeholder can never + // mis-pin `brain.dimensions` for it. A plugin-registered native + // 'embeddings' provider has no cold-compile cost AND may use a + // different dimension, so it keeps embedding the root for real (same + // as before this fix) rather than risk pinning the wrong dimension + // ahead of the caller's own first real embed. + const rootVector = this.brain.usesDefaultWasmEmbedder() + ? new Array(VirtualFileSystem.VFS_ROOT_VECTOR_DIMENSIONS).fill(0) + : undefined + await this.brain.add({ id: rootId, // Fixed ID - storage ensures uniqueness data: '/', @@ -271,7 +315,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { // public AddParams.visibility union ('public' | 'internal') — this is the single // sanctioned internal setter, hence the cast. visibility: 'system' as 'public' | 'internal', - metadata: this.getRootMetadata() + metadata: this.getRootMetadata(), + ...(rootVector ? { vector: rootVector } : {}) }) return rootId diff --git a/tests/unit/brainy/open-path.test.ts b/tests/unit/brainy/open-path.test.ts new file mode 100644 index 00000000..3556d7a5 --- /dev/null +++ b/tests/unit/brainy/open-path.test.ts @@ -0,0 +1,199 @@ +/** + * OPEN-PATH tests: init() must never gate on the embedding model, the VFS + * root bootstrap must never touch the embedding engine, and a slow open + * must narrate its phases. + * + * Background: a production restart storm measured 90,017ms for a single + * brain init vs 1,117ms quiet — an ~80x contention multiplier — traced to + * every writer's init() eagerly awaiting the process-global WASM embedding + * engine before the VFS root even existed. See src/brainy.ts performInit() + * and src/vfs/VirtualFileSystem.ts doInitializeRoot(). + */ + +import { describe, it, expect, vi } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage' +import { embeddingManager } from '../../../src/embeddings/EmbeddingManager' +import { createTestConfig } from '../../helpers/test-factory' + +/** + * The four signals `isDeterministicEmbedMode()` checks (see + * src/embeddings/deterministicEmbedMode.ts). The global unit-test setup + * (tests/setup-unit.ts) sets some of these for the whole file/run in some + * vitest configurations; other configurations leave them unset and run the + * real WASM engine instead. The background-warm tests below need the + * "not unit-test mode" branch of performInit() to actually execute, so they + * save/clear/restore all four explicitly — deterministic regardless of + * which config invoked this file, never relying on ambient state. + */ +function withRealEmbedderBranch(fn: () => Promise): Promise { + const savedEnvDeterministic = process.env.BRAINY_DETERMINISTIC_EMBEDDINGS + const savedEnvUnitTest = process.env.BRAINY_UNIT_TEST + const g = globalThis as Record + const savedGlobalDeterministic = g.__BRAINY_DETERMINISTIC_EMBED__ + const savedGlobalUnitTest = g.__BRAINY_UNIT_TEST__ + + delete process.env.BRAINY_DETERMINISTIC_EMBEDDINGS + delete process.env.BRAINY_UNIT_TEST + delete g.__BRAINY_DETERMINISTIC_EMBED__ + delete g.__BRAINY_UNIT_TEST__ + + const restore = () => { + if (savedEnvDeterministic !== undefined) process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = savedEnvDeterministic + if (savedEnvUnitTest !== undefined) process.env.BRAINY_UNIT_TEST = savedEnvUnitTest + if (savedGlobalDeterministic !== undefined) g.__BRAINY_DETERMINISTIC_EMBED__ = savedGlobalDeterministic + if (savedGlobalUnitTest !== undefined) g.__BRAINY_UNIT_TEST__ = savedGlobalUnitTest + } + + return fn().finally(restore) +} + +/** + * A MemoryStorage whose init() takes an artificially long time — a + * controllable fake seam (not a wall-clock race) that reliably pushes + * performInit()'s "storage-init" phase (and therefore the total open time) + * past the 2000ms narration threshold, without touching the filesystem or + * relying on real contention. + */ +class SlowMemoryStorage extends MemoryStorage { + override async init(): Promise { + await new Promise((resolve) => setTimeout(resolve, 2200)) + await super.init() + } +} + +describe('OPEN-PATH: init() never gates on the embedding model', () => { + it('bootstrapping a fresh store never calls the embedding engine (VFS root add is engine-untouched)', async () => { + const embedSpy = vi.spyOn(embeddingManager, 'embed') + const brain = new Brainy(createTestConfig()) + try { + await brain.init() + + // The VFS root's add() must never have reached the embedding engine — + // it carries an explicit placeholder vector instead (see + // VirtualFileSystem.doInitializeRoot()). + expect(embedSpy).not.toHaveBeenCalled() + + // Sanity: the VFS is genuinely usable afterwards. + const files = await brain.vfs.readdir('/') + expect(files).toEqual([]) + } finally { + await brain.close() + embedSpy.mockRestore() + } + }) + + it('starts the embedding-engine warm in the BACKGROUND — init() resolves before the warm does', async () => { + await withRealEmbedderBranch(async () => { + const events: string[] = [] + let releaseWarm!: () => void + const warmGate = new Promise((resolve) => { + releaseWarm = resolve + }) + + const initSpy = vi.spyOn(embeddingManager, 'init').mockImplementation(async () => { + events.push('warm-start') + await warmGate + events.push('warm-resolve') + }) + + const brain = new Brainy(createTestConfig()) + try { + await brain.init() + events.push('init-resolved') + + // init() started the warm but returned WITHOUT waiting for it. + expect(initSpy).toHaveBeenCalledTimes(1) + expect(events).toEqual(['warm-start', 'init-resolved']) + + // Now let the fake warm finish and confirm it lands strictly after. + releaseWarm() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(events).toEqual(['warm-start', 'init-resolved', 'warm-resolve']) + } finally { + await brain.close() + initSpy.mockRestore() + } + }) + }) + + it('narrates a background warm FAILURE loudly instead of losing it silently', async () => { + await withRealEmbedderBranch(async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const initSpy = vi + .spyOn(embeddingManager, 'init') + .mockRejectedValue(new Error('simulated cold-compile failure')) + + const brain = new Brainy(createTestConfig()) + try { + // init() itself must still resolve — a failed background warm is + // never fatal to open(). + await expect(brain.init()).resolves.toBeUndefined() + + // Give the background .catch() a microtask/macrotask to run. + await new Promise((resolve) => setTimeout(resolve, 0)) + + const failureLine = warnSpy.mock.calls + .map((args) => args.map(String).join(' ')) + .find((line) => line.includes('background embedding-engine warm FAILED')) + expect(failureLine).toBeDefined() + expect(failureLine).toContain('simulated cold-compile failure') + } finally { + await brain.close() + initSpy.mockRestore() + warnSpy.mockRestore() + } + }) + }) + + it('eagerEmbeddings: false starts no warm at all', async () => { + await withRealEmbedderBranch(async () => { + const initSpy = vi.spyOn(embeddingManager, 'init') + const brain = new Brainy({ ...createTestConfig(), eagerEmbeddings: false }) + try { + await brain.init() + expect(initSpy).not.toHaveBeenCalled() + } finally { + await brain.close() + initSpy.mockRestore() + } + }) + }) + + it('narrates a slow open with a per-phase ms breakdown once total time exceeds 2000ms', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const brain = new Brainy({ ...createTestConfig(), storage: new SlowMemoryStorage() }) + try { + await brain.init() + + const slowOpenLine = warnSpy.mock.calls + .map((args) => args.map(String).join(' ')) + .find((line) => line.includes('[Brainy] slow open:')) + + expect(slowOpenLine).toBeDefined() + expect(slowOpenLine).toContain('storage-init=') + expect(slowOpenLine).toContain('generation-store-open-fold=') + expect(slowOpenLine).toContain('index-init-gate=') + expect(slowOpenLine).toContain('vfs-bootstrap=') + expect(slowOpenLine).toContain('embedding-warm-started=') + } finally { + await brain.close() + warnSpy.mockRestore() + } + }, 20000) + + it('stays silent about phase timing when open is fast (under 2000ms)', async () => { + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const brain = new Brainy(createTestConfig()) + try { + await brain.init() + const slowOpenLine = warnSpy.mock.calls + .map((args) => args.map(String).join(' ')) + .find((line) => line.includes('[Brainy] slow open:')) + expect(slowOpenLine).toBeUndefined() + } finally { + await brain.close() + warnSpy.mockRestore() + } + }) +}) From fc516da6eb38636a071ade83b5db459f5e0e2faa Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 10:10:01 -0700 Subject: [PATCH 012/150] =?UTF-8?q?feat(vfs):=20implement=20readdir's=20re?= =?UTF-8?q?cursive=20option=20=E2=80=94=20typed=20since=207.30,=20never=20?= =?UTF-8?q?read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vfs.readdir()'s ReaddirOptions.recursive was typed but silently ignored: a recursive request behaved identically to a non-recursive one. Implemented properly: recursive listing walks every descendant (files and directories, any depth) via the same graph-traversal + one-batch-fetch path getTreeStructure()/getDescendants() already use, and reports each entry as a path relative to the queried directory (Node's fs.readdir(dir, { recursive: true }) convention) — 'sub/file.txt', not just 'file.txt'. With withFileTypes: true, each VFSDirent.name carries that same relative path when recursive (matching the string-array form byte for byte); VFSDirent.path stays the absolute VFS path either way, so no information is lost. Filter/sort/pagination compose unchanged, now over the full recursive set. Non-recursive behavior (direct children, named by basename) is unchanged. --- src/vfs/VirtualFileSystem.ts | 39 ++++++++-- src/vfs/types.ts | 21 ++++- tests/unit/vfs-readdir-recursive.test.ts | 99 ++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 tests/unit/vfs-readdir-recursive.test.ts diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 0a000396..90018863 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -1274,7 +1274,20 @@ export class VirtualFileSystem implements IVirtualFileSystem { } /** - * Read directory contents + * @description List a directory's contents. Non-recursive (default) + * returns direct children only, named by basename. `recursive: true` + * lists every descendant at any depth (files and directories), each + * reported as a path RELATIVE TO THE QUERIED DIRECTORY — matching Node's + * `fs.readdir(dir, { recursive: true })` convention — e.g. `'sub'` and + * `'sub/file.txt'` for a nested file. With `withFileTypes: true`, each + * {@link VFSDirent}'s `name` carries that same value (relative when + * recursive, basename otherwise); `path` is always the absolute VFS path + * either way. + * @param path - The directory to list. + * @param options - `recursive`, `withFileTypes`, `filter`, `sort`, + * `offset`/`limit` (pagination applies AFTER filter/sort, over the full + * recursive set when `recursive: true`). + * @throws {VFSError} ENOTDIR when `path` is not a directory. */ async readdir(path: string, options?: ReaddirOptions): Promise { await this.ensureInitialized() @@ -1287,8 +1300,12 @@ export class VirtualFileSystem implements IVirtualFileSystem { throw new VFSError(VFSErrorCode.ENOTDIR, `Not a directory: ${path}`, path, 'readdir') } - // Get children - let children = await this.pathResolver.getChildren(entityId) + // Direct children, or every descendant at any depth. gatherDescendants() + // is the same graph-traversal + ONE-batch-fetch path getTreeStructure()/ + // getDescendants() already use — no per-directory storage round trips. + let children = options?.recursive + ? await this.gatherDescendants(entityId, Infinity) + : await this.pathResolver.getChildren(entityId) // Apply filters if (options?.filter) { @@ -1312,17 +1329,29 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Directory access time updates caused 50-100ms GCS write on EVERY readdir // await this.updateAccessTime(entityId) // ← REMOVED + // The queried directory's own canonical (already-normalized) path — the + // base every recursive entry's relative name is computed against. Using + // the resolved entity's OWN path (rather than the raw `path` argument) + // means no separate normalization step is needed here. + const baseDir = entity.metadata.path + const relativeToBase = (childPath: string): string => { + const prefix = baseDir === '/' ? '/' : `${baseDir}/` + return childPath.startsWith(prefix) ? childPath.slice(prefix.length) : childPath + } + // Return appropriate format if (options?.withFileTypes) { return children.map(child => ({ - name: child.metadata.name, + name: options?.recursive ? relativeToBase(child.metadata.path) : child.metadata.name, path: child.metadata.path, type: child.metadata.vfsType, entityId: child.id } as VFSDirent)) } - return children.map(child => child.metadata.name) + return children.map(child => + options?.recursive ? relativeToBase(child.metadata.path) : child.metadata.name + ) } // ============= Metadata Operations ============= diff --git a/src/vfs/types.ts b/src/vfs/types.ts index 9188476b..17687dd3 100644 --- a/src/vfs/types.ts +++ b/src/vfs/types.ts @@ -133,8 +133,17 @@ export interface VFSStats { * Directory entry (for readdir) */ export interface VFSDirent { + /** + * The entry's basename (e.g. `'file.txt'`) when `readdir()` was called + * WITHOUT `recursive: true`. When `recursive: true` was set, this is + * instead the entry's path RELATIVE TO THE QUERIED DIRECTORY (e.g. + * `'sub/file.txt'` for a nested file) — the same value that would appear + * in the plain string-array form of a recursive `readdir()` call. `path` + * below always carries the absolute VFS path regardless of `recursive`, + * so nothing is lost either way. + */ name: string - path: string // Full path + path: string // Full (absolute) VFS path — always absolute, recursive or not type: 'file' | 'directory' | 'symlink' entityId: string // Underlying entity ID } @@ -240,7 +249,15 @@ export interface ReaddirOptions { withFileTypes?: boolean // Return Dirent objects // VFS-specific options - recursive?: boolean // Include subdirectories + /** + * List every descendant (files and directories, all depths), not just + * direct children. Entries are reported as paths RELATIVE TO THE QUERIED + * DIRECTORY (Node's `fs.readdir(dir, { recursive: true })` convention) — + * a string-array result contains e.g. `'sub/file.txt'`, and with + * `withFileTypes: true` each `VFSDirent.name` carries that same relative + * path (see {@link VFSDirent}). Default: `false` (direct children only). + */ + recursive?: boolean limit?: number // Max results offset?: number // Skip N results cursor?: string // Pagination cursor diff --git a/tests/unit/vfs-readdir-recursive.test.ts b/tests/unit/vfs-readdir-recursive.test.ts new file mode 100644 index 00000000..2ee8a775 --- /dev/null +++ b/tests/unit/vfs-readdir-recursive.test.ts @@ -0,0 +1,99 @@ +/** + * vfs.readdir()'s `recursive` option: typed since 7.30 but never read, so it + * silently behaved exactly like `recursive: false`. This pins the real, + * documented contract: a recursive listing returns every descendant (files + * AND directories, all depths) as paths RELATIVE TO THE QUERIED DIRECTORY — + * the same convention Node's `fs.readdir(dir, { recursive: true })` uses — + * for both the plain string-array form and the `withFileTypes` VFSDirent + * form (whose `name` carries that same relative path when recursive). + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import type { VFSDirent } from '../../src/vfs/types.js' + +describe('vfs.readdir() recursive option', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'memory' }, + silent: true + }) + await brain.init() + + // Build: + // /a/b.txt + // /a/sub/c.txt + // /a/sub/deeper/d.txt + // /a/sub2/ (empty directory) + await brain.vfs.writeFile('/a/b.txt', 'B') + await brain.vfs.writeFile('/a/sub/c.txt', 'C') + await brain.vfs.writeFile('/a/sub/deeper/d.txt', 'D') + await brain.vfs.mkdir('/a/sub2', { recursive: true }) + }) + + afterEach(async () => { + await brain.close() + }) + + it('non-recursive (default) still returns only direct children, by basename', async () => { + const entries = await brain.vfs.readdir('/a') as string[] + expect([...entries].sort()).toEqual(['b.txt', 'sub', 'sub2']) + }) + + it('recursive: true returns every descendant as a path relative to the queried directory', async () => { + const entries = await brain.vfs.readdir('/a', { recursive: true }) as string[] + expect([...entries].sort()).toEqual([ + 'b.txt', + 'sub', + 'sub/c.txt', + 'sub/deeper', + 'sub/deeper/d.txt', + 'sub2' + ]) + }) + + it('recursive: true at the root has no leading slash on relative entries', async () => { + const entries = await brain.vfs.readdir('/', { recursive: true }) as string[] + expect(entries).toContain('a') + expect(entries).toContain('a/b.txt') + expect(entries).toContain('a/sub/deeper/d.txt') + for (const entry of entries) { + expect(entry.startsWith('/')).toBe(false) + } + }) + + it('recursive + withFileTypes: VFSDirent.name is the relative path, .path stays absolute', async () => { + const entries = await brain.vfs.readdir('/a', { + recursive: true, + withFileTypes: true + }) as VFSDirent[] + + const byName = new Map(entries.map((e) => [e.name, e])) + + const nested = byName.get('sub/deeper/d.txt') + expect(nested).toBeDefined() + expect(nested!.path).toBe('/a/sub/deeper/d.txt') + expect(nested!.type).toBe('file') + + const nestedDir = byName.get('sub/deeper') + expect(nestedDir).toBeDefined() + expect(nestedDir!.path).toBe('/a/sub/deeper') + expect(nestedDir!.type).toBe('directory') + + // Non-recursive VFSDirent behavior is unchanged: name is the basename. + const direct = await brain.vfs.readdir('/a', { withFileTypes: true }) as VFSDirent[] + const directEntry = direct.find((e) => e.path === '/a/b.txt') + expect(directEntry?.name).toBe('b.txt') + }) + + it('recursive + filter composes: only files survive a type filter', async () => { + const entries = await brain.vfs.readdir('/a', { + recursive: true, + filter: { type: 'file' } + }) as string[] + expect([...entries].sort()).toEqual(['b.txt', 'sub/c.txt', 'sub/deeper/d.txt']) + }) +}) From 258e9042afb8dbc1e8ed2f91bf532d0692b0a995 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 10:10:19 -0700 Subject: [PATCH 013/150] fix(add): empty string is real data, not a missing field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateAddParams() treated '' as falsy and rejected it with "Missing required field 'data'" — so a legitimate empty file's first write always failed. Only null/undefined data (with no vector either) is genuinely absent; '' is real content. Fixed the check, plus the identical bug in validateUpdateParams() (truncating a file to empty via overwrite hit the same falsy check) and in update()/transact()'s update planner, where a plain `Boolean(params.data)`/truthy check on the resolved vector would have silently skipped both the deferred-embed marker and the eager re-embed for an emptied value — a stale vector with no path to ever correct itself. Verified end-to-end: vfs.writeFile('/empty.txt', '') now succeeds, readFile() returns '', the file lists, and stat() reports size 0; the existing "should reject empty string as data" tests (unit + integration) asserted the old buggy behavior and are updated to assert the fixed contract instead. --- src/brainy.ts | 24 +++++++--- src/utils/paramValidation.ts | 19 +++++--- .../brainy-core.integration.test.ts | 12 +++-- tests/unit/brainy/add.test.ts | 19 +++++--- tests/unit/utils/paramValidation.test.ts | 45 ++++++++++++++++++- tests/vfs/vfs.unit.test.ts | 29 ++++++++++++ 6 files changed, 128 insertions(+), 20 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 9cd7dcb4..3791f768 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -3562,11 +3562,20 @@ export class Brainy implements BrainyInterface { // new `data`); otherwise new `data` re-embeds; otherwise the existing // vector is kept. Any vector change re-indexes HNSW below. let vector = existing.vector + // 'data' is a real new value whenever it's not null/undefined — an + // empty string ('') is legitimate content (e.g. truncating a file to + // empty via overwrite), matching validateUpdateParams's absent-vs-empty + // distinction. Using `Boolean(params.data)` here would treat '' as "no + // new data", silently skipping BOTH the deferred marker and the eager + // re-embed below — a stale vector left behind with no path to ever + // correct itself (a quiet loss, not the deferred-but-eventually- + // correct flicker the deferEmbedding contract promises). + const hasNewData = params.data !== undefined && params.data !== null // MT5 deferred re-embedding: the OLD vector keeps serving semantic // search — stale-but-present, never absent (the flicker law) — until // the background worker embeds the new data and swaps it atomically. const deferringEmbed = - params.deferEmbedding === true && Boolean(params.data) && !params.vector + params.deferEmbedding === true && hasNewData && !params.vector if (params.vector) { if (this.dimensions && params.vector.length !== this.dimensions) { throw new Error( @@ -3574,13 +3583,13 @@ export class Brainy implements BrainyInterface { ) } vector = params.vector - } else if (params.data && !deferringEmbed) { + } else if (hasNewData && !deferringEmbed) { vector = await this.embed(params.data) } // A deferred data change does NOT reindex now (the vector is unchanged; // the worker's atomic swap carries the real reindex later). const needsReindexing = Boolean( - (params.data && !deferringEmbed) || params.type || params.vector + (hasNewData && !deferringEmbed) || params.type || params.vector ) // Always update the noun with new metadata @@ -10501,6 +10510,11 @@ export class Brainy implements BrainyInterface { // Resolve the updated vector — mirror of update(): an explicit `vector` // always wins, new `data` re-embeds, otherwise the existing vector is // kept. Any vector change re-indexes HNSW below. + // 'data' is present whenever it's not null/undefined — '' is real + // content (see the identical hasNewData in update()); a plain truthy + // check would silently skip re-embedding an emptied value and leave a + // stale vector with no path to ever correct itself. + const hasNewData = params.data !== undefined && params.data !== null let vector = existing.vector if (params.vector) { if (this.dimensions && params.vector.length !== this.dimensions) { @@ -10509,10 +10523,10 @@ export class Brainy implements BrainyInterface { ) } vector = params.vector - } else if (params.data) { + } else if (hasNewData) { vector = await this.embed(params.data) } - const needsReindexing = Boolean(params.data || params.type || params.vector) + const needsReindexing = Boolean(hasNewData || params.type || params.vector) const newMetadata = params.merge !== false diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index b8036746..d43559dc 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -540,6 +540,11 @@ function rejectForgedSystemKeys(metadata: Record | undefined, s export function validateAddParams(params: AddParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'add()') + // 'data' is ABSENT only when null/undefined — an empty string ('') is real + // content (a legitimate empty file's first write) and must not be treated + // as missing. Falsy-but-present values (0, false, '') all count as present; + // only the true "nothing was given" case is absent. + const hasData = params.data !== undefined && params.data !== null // MT5 deferred embedding: an explicit vector has nothing to defer, and a // deferral without data has nothing to embed — both are caller bugs that // must refuse with the fix, never be silently reinterpreted. @@ -550,14 +555,14 @@ export function validateAddParams(params: AddParams): void { `the vector is already computed; drop one of the two.` ) } - if (!params.data) { + if (!hasData) { throw new Error( `add(): deferEmbedding requires 'data' (the content the background worker will embed).` ) } } // Universal truth: must have data or vector - if (!params.data && !params.vector) { + if (!hasData && !params.vector) { throw new Error( `Invalid add() parameters: Missing required field 'data'\n` + `\nReceived: ${JSON.stringify({ @@ -597,6 +602,10 @@ export function validateAddParams(params: AddParams): void { */ export function validateUpdateParams(params: UpdateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'update()') + // Same absent-vs-empty distinction as validateAddParams: '' is a real new + // value (e.g. truncating a file to empty content via overwrite), only + // null/undefined means "no new data was given". + const hasData = params.data !== undefined && params.data !== null if ((params as UpdateParams & { deferEmbedding?: boolean }).deferEmbedding === true) { if (params.vector) { throw new Error( @@ -604,7 +613,7 @@ export function validateUpdateParams(params: UpdateParams): void { `the vector is already computed; drop one of the two.` ) } - if (!params.data) { + if (!hasData) { throw new Error( `update(): deferEmbedding requires new 'data' — without a data change there is nothing to re-embed.` ) @@ -614,10 +623,10 @@ export function validateUpdateParams(params: UpdateParams): void { if (!params.id) { throw new Error('id is required for update') } - + // Universal truth: must update something if ( - !params.data && + !hasData && !params.metadata && !params.type && !params.vector && diff --git a/tests/integration/brainy-core.integration.test.ts b/tests/integration/brainy-core.integration.test.ts index dd04703e..e5b01caf 100644 --- a/tests/integration/brainy-core.integration.test.ts +++ b/tests/integration/brainy-core.integration.test.ts @@ -337,12 +337,18 @@ describe('Brainy 3.0 Core (Integration Tests - Real AI)', () => { describe('Error Handling and Edge Cases', () => { it('should handle invalid inputs gracefully', async () => { - // Empty data is rejected with a clear validation error (8.0 requires a - // non-empty `data` or a `vector` — empty string carries no signal to embed). + // Empty string is REAL content (e.g. an empty file's first write), not + // a missing field — only null/undefined data (with no vector either) + // is rejected. See src/utils/paramValidation.ts validateAddParams(). await expect(brain.add({ data: '', type: 'document' - })).rejects.toThrow(/data/) + })).resolves.toBeDefined() + + // Missing BOTH data and vector is still the real "nothing to embed" error. + await expect(brain.add({ + type: 'document' + } as any)).rejects.toThrow(/data/) // Test with very long text — valid input, resolves to an id. const longText = 'Lorem ipsum '.repeat(10000) diff --git a/tests/unit/brainy/add.test.ts b/tests/unit/brainy/add.test.ts index 10690e2f..7203690c 100644 --- a/tests/unit/brainy/add.test.ts +++ b/tests/unit/brainy/add.test.ts @@ -335,15 +335,24 @@ describe('Brainy.add()', () => { }) describe('edge cases', () => { - it('should reject empty string as data', async () => { - // Arrange + it('should accept an empty string as real (empty) data', async () => { + // Arrange — '' is legitimate content (e.g. an empty file's first + // write), not a missing field. Only null/undefined data (with no + // vector either) is "missing" — see the separate + // 'data and vector are both missing' test above. const params = createAddParams({ data: '', type: 'thing' }) - - // Act & Assert - Empty string is not valid data - await expect(brain.add(params)).rejects.toThrow('Invalid add() parameters: Missing required field \'data\'') + + // Act + const id = await brain.add(params) + + // Assert — stored and readable back as empty, not rejected + expect(id).toBeDefined() + const entity = await brain.get(id) + expect(entity).not.toBeNull() + expect(entity!.data).toBe('') }) it('should handle very long text content', async () => { diff --git a/tests/unit/utils/paramValidation.test.ts b/tests/unit/utils/paramValidation.test.ts index 7e5212b8..805dd40d 100644 --- a/tests/unit/utils/paramValidation.test.ts +++ b/tests/unit/utils/paramValidation.test.ts @@ -149,7 +149,33 @@ describe('Zero-Config Parameter Validation', () => { type: NounType.Document } as AddParams)).toThrow('Invalid add() parameters: Missing required field \'data\'') }) - + + it('should accept an empty string as real data — only null/undefined is "missing"', () => { + // A legitimate empty file's first write: '' is content, not absence. + expect(() => validateAddParams({ + data: '', + type: NounType.Document + })).not.toThrow() + + // null/undefined (with no vector) is still the genuine missing-field case. + expect(() => validateAddParams({ + data: null as any, + type: NounType.Document + })).toThrow('Invalid add() parameters: Missing required field \'data\'') + expect(() => validateAddParams({ + data: undefined, + type: NounType.Document + })).toThrow('Invalid add() parameters: Missing required field \'data\'') + }) + + it('deferEmbedding accepts empty-string data (real content, not absence)', () => { + expect(() => validateAddParams({ + data: '', + type: NounType.Document, + deferEmbedding: true + } as AddParams)).not.toThrow() + }) + it('should validate NounType', () => { expect(() => validateAddParams({ data: 'test', @@ -190,7 +216,22 @@ describe('Zero-Config Parameter Validation', () => { id: 'test-id' })).toThrow('must specify at least one field to update') }) - + + it('empty-string data counts as a real field to update (truncating content)', () => { + expect(() => validateUpdateParams({ + id: 'test-id', + data: '' + })).not.toThrow() + }) + + it('deferEmbedding accepts empty-string data on update', () => { + expect(() => validateUpdateParams({ + id: 'test-id', + data: '', + deferEmbedding: true + } as UpdateParams)).not.toThrow() + }) + it('should validate NounType if changing', () => { expect(() => validateUpdateParams({ id: 'test-id', diff --git a/tests/vfs/vfs.unit.test.ts b/tests/vfs/vfs.unit.test.ts index 5ea79377..4b4ba8d2 100644 --- a/tests/vfs/vfs.unit.test.ts +++ b/tests/vfs/vfs.unit.test.ts @@ -53,6 +53,35 @@ describe('VirtualFileSystem - Production Tests', () => { expect(exists).toBe(true) }) + it('should write and read an empty (0-byte) file end-to-end', async () => { + // Pin: validateAddParams() used to treat '' as a missing 'data' field + // (falsy check), so a legitimate empty file's FIRST write threw + // "Missing required field 'data'". '' is real content, not an absent + // field — only null/undefined is absent. + const path = '/empty.txt' + + await vfs.writeFile(path, '') + + const result = await vfs.readFile(path) + expect(result.toString()).toBe('') + + const exists = await vfs.exists(path) + expect(exists).toBe(true) + + const stats = await vfs.stat(path) + expect(stats.size).toBe(0) + expect(stats.isFile()).toBe(true) + + // The file lists like any other. + const entries = await vfs.readdir('/') as string[] + expect(entries).toContain('empty.txt') + + // Overwriting it back to empty (truncate) must also succeed. + await vfs.writeFile(path, 'not empty anymore') + await vfs.writeFile(path, '') + expect((await vfs.readFile(path)).toString()).toBe('') + }) + it('should handle binary files', async () => { const binaryData = Buffer.from([0x00, 0x01, 0x02, 0xFF]) const path = '/binary.dat' From ddd5e71928e4ba826434ed384fe6fea05eca73f3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 10:47:51 -0700 Subject: [PATCH 014/150] fix(storage): an unknown nested storage config can never silently land on the shared default root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The factory loudly rejects every REMOVED pre-8.0 path key, but an unknown nested `config` object (e.g. `storage: { config: { baseDir } }` — a shape that was never supported) fell through SILENTLY to the zero-config default directory. Every instance constructed with such a shape wrote to ONE shared on-disk root while its caller believed each had its own — found live when two integration tests' brains shared a store across an entire single-process CI run and a health probe refused on the foreign edges it sampled. A nested `config` carrying any path-shaped key now throws the same loud migration error, naming the canonical `path` rename. The two tests are repaired to the supported shape (and now actually test isolated stores, for the first time since 8.0). --- src/storage/storageFactory.ts | 15 +++++++++++++++ .../integration/batchImportWithRelations.test.ts | 8 +------- tests/integration/readAfterWrite.test.ts | 8 +------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/storage/storageFactory.ts b/src/storage/storageFactory.ts index 64b18dc1..46c67f44 100644 --- a/src/storage/storageFactory.ts +++ b/src/storage/storageFactory.ts @@ -154,6 +154,21 @@ export function resolveFilesystemRoot( ) { throwRemovedStorageKey('fileSystemStorage.path') } + // A nested `config` object carrying a path-shaped key is the same hazard in + // a shape nobody ever supported: it used to fall through SILENTLY to the + // shared default root — every instance writing one directory while its + // caller believed each had its own. (Found live: an integration test's + // brains shared one store across a whole single-process run and a health + // probe refused on the foreign edges it sampled.) Loud, with the rename. + const nested = (config as Record).config + if (nested && typeof nested === 'object') { + const pathish = ['path', 'baseDir', 'rootDirectory', 'rootDir', 'dir', 'directory'] + const hit = pathish.find( + (k) => typeof (nested as Record)[k] === 'string' && + ((nested as Record)[k] as string).length > 0 + ) + if (hit) throwRemovedStorageKey(`config.${hit}`) + } // 3. Zero-config default. A `type: 'filesystem'` with no path lands here // intentionally ("persist, default location"). diff --git a/tests/integration/batchImportWithRelations.test.ts b/tests/integration/batchImportWithRelations.test.ts index 7fe8e511..4095b51c 100644 --- a/tests/integration/batchImportWithRelations.test.ts +++ b/tests/integration/batchImportWithRelations.test.ts @@ -15,13 +15,7 @@ describe('Batch Import with Immediate Relations (v5.7.3 Fix)', () => { // Initialize brain brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - config: { - baseDir: testDir, - enableCompression: false // Faster tests - } - }, + storage: { type: 'filesystem', path: testDir }, dimensions: 384 }) diff --git a/tests/integration/readAfterWrite.test.ts b/tests/integration/readAfterWrite.test.ts index e0ab5863..cf1dc9ec 100644 --- a/tests/integration/readAfterWrite.test.ts +++ b/tests/integration/readAfterWrite.test.ts @@ -34,13 +34,7 @@ describe('Read-After-Write Consistency (v5.7.2 Bug Fix)', () => { testDir = join(tmpdir(), `brainy-consistency-${Date.now()}-${Math.random().toString(36).substring(7)}`) brain = new Brainy({ requireSubtype: false, - storage: { - type: 'filesystem', - config: { - baseDir: testDir, - enableCompression: false // Faster tests - } - }, + storage: { type: 'filesystem', path: testDir }, dimensions: 384 }) From 553e0d97ae09cb7d6b73b05678124a6587c682e8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 10:47:51 -0700 Subject: [PATCH 015/150] feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit repairIndex() acted only on heal:'rebuild' — an invariant asking for the INCREMENTAL heal (re-post exactly what the ledger names, O(missing), never a store-sized rebuild) did nothing on brainy's side. A failing 'repair' verdict now routes to the provider's feature-detected repair(); the post-heal RE-READ of the report decides success (the acceptance meta-pin's law — run the named heal once, re-read, nothing may still fail the same way), and a repair that does not converge is recorded with the escalation named: repairIndex({ rebuild: [family] }). --- src/brainy.ts | 42 ++++++++++++++++++++++++- tests/integration/repair-report.test.ts | 42 +++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/brainy.ts b/src/brainy.ts index 5a600402..d5f2fd84 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -17549,10 +17549,50 @@ export class Brainy implements BrainyInterface { } else { await p.rebuild() } + } else if ( + report.invariants.some((i) => !i.holds && i.heal === 'repair') && + typeof (provider as { repair?: () => Promise }).repair === 'function' + ) { + // INCREMENTAL HEAL ROUTING (ADR-008 D4): a failing verdict whose heal + // is 'repair' routes to the provider's own repair() — O(missing), + // re-posting exactly what its ledger names, never a store-sized + // rebuild. The return shape is the provider's own; the RE-READ of the + // report is what decides success (the acceptance meta-pin's law: run + // the named heal once, re-read, nothing may still fail the same way). + const failingRepairs = report.invariants + .filter((i) => !i.holds && i.heal === 'repair') + .map((i) => i.name) + prodLog.warn( + `[Brainy] repairIndex(): provider '${report.provider}' asks for an incremental ` + + `repair (${failingRepairs.join(', ')}) — running its own repair().` + ) + await (provider as { repair: () => Promise }).repair() + let cleared = false + let after: ProviderInvariantReport | null = null + try { + after = await p.validateInvariants() + cleared = !after.invariants.some( + (i) => !i.holds && i.heal === 'repair' && failingRepairs.includes(i.name) + ) + } catch { + // The post-heal re-read failing is itself reportable, never a crash. + } + record(`provider:${report.provider}`, { + checked: true, + healed: cleared ? failingRepairs.length : 0, + detail: cleared + ? `incremental repair cleared: ${failingRepairs.join(', ')}` + : `repair() ran but the re-read still fails (${ + after + ? after.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ') + : 're-read threw' + }) — escalate to repairIndex({ rebuild: ['${familyName}'] })`, + reason: cleared ? undefined : 'repair did not converge' + }) } else { record(`provider:${report.provider}`, { checked: true, healed: 0, - detail: `unhealthy without a rebuild verdict (failing: ${report.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ')})` + detail: `unhealthy without a routable verdict (failing: ${report.invariants.filter((i) => !i.holds).map((i) => `${i.name}→${i.heal}`).join(', ')})` }) } } diff --git a/tests/integration/repair-report.test.ts b/tests/integration/repair-report.test.ts index 273eee2e..28e0ad99 100644 --- a/tests/integration/repair-report.test.ts +++ b/tests/integration/repair-report.test.ts @@ -68,4 +68,46 @@ describe('repairIndex per-family receipt', () => { expect(orphans!.healed, 'the ghost was pruned and receipted').toBeGreaterThan(0) expect(report.healedTotal).toBeGreaterThan(0) }, 120000) + + + it("a heal:'repair' verdict routes to the provider's own repair(), and the re-read decides", async () => { + // A fake provider report: one failing invariant asking for the INCREMENTAL + // heal. repairIndex must call repair() (never rebuild()) and count the heal + // only when the post-repair re-read clears the same verdict. + const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-route-')) + const brain: any = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false, silent: true }) + await brain.init() + brains.push(brain) + let repairCalls = 0 + let rebuildCalls = 0 + let healed = false + const failing = { + provider: 'vector', healthy: false, serving: true, + invariants: [{ name: 'node-coverage', holds: false, detail: 'short 3', heal: 'repair' as const }], + checkedAt: 1, durationMs: 1 + } + const clean = { + provider: 'vector', healthy: true, serving: true, + invariants: [{ name: 'node-coverage', holds: true, detail: 'ok', heal: 'none' as const }], + checkedAt: 2, durationMs: 1 + } + ;(brain.index as any).validateInvariants = async () => (healed ? clean : failing) + ;(brain.index as any).repair = async () => { repairCalls++; healed = true; return { repaired: 3 } } + const origRebuild = (brain.index as any).rebuild + ;(brain.index as any).rebuild = async () => { rebuildCalls++ } + try { + const report = await brain.repairIndex() + const row = report.families.find((f: any) => f.family === 'provider:vector') + expect(row, 'the provider family is in the receipt').toBeDefined() + expect(repairCalls, 'repair() ran exactly once').toBe(1) + expect(rebuildCalls, "a heal:'repair' verdict never runs rebuild()").toBe(0) + expect(row!.healed, 'the cleared verdict counts as healed').toBe(1) + expect(String(row!.detail)).toMatch(/incremental repair cleared: node-coverage/) + } finally { + delete (brain.index as any).validateInvariants + delete (brain.index as any).repair + ;(brain.index as any).rebuild = origRebuild + } + }) + }) From 39b916a3c0c011fe8a4968947c35df86b503f2f0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 11:04:30 -0700 Subject: [PATCH 016/150] =?UTF-8?q?test(readiness):=20the=20report=20helpe?= =?UTF-8?q?r's=20clock=20freezes=20=E2=80=94=20two=20independently-built?= =?UTF-8?q?=20reports=20compared=20across=20a=20millisecond=20tick=20made?= =?UTF-8?q?=20the=20plant=20lane=20red?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/unit/utils/indexReadiness.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/indexReadiness.test.ts b/tests/unit/utils/indexReadiness.test.ts index e00d1ff0..75504c71 100644 --- a/tests/unit/utils/indexReadiness.test.ts +++ b/tests/unit/utils/indexReadiness.test.ts @@ -29,7 +29,11 @@ function report(overrides: Partial = {}): HealthReport { healthy: true, serving: true, invariants: [], - checkedAt: Date.now(), + // A FIXED stamp, never Date.now(): the pin at :98 compares two + // independently-built reports, and a live clock made them differ by 1ms + // whenever the millisecond ticked between the two calls — a plant-lane + // red that had nothing to do with the code under test. + checkedAt: 1_700_000_000_000, durationMs: 1, generation: 1, unledgered: [], From 0e1286e321192363169ca3e8c0b0d639b72d0ecd Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 11:38:06 -0700 Subject: [PATCH 017/150] chore(release): 10.4.0-rc.2 --- CHANGELOG.md | 11 +++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd64d88f..f4867f53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.0-rc.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) + +- test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) +- feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) +- fix(storage): an unknown nested storage config can never silently land on the shared default root (ddd5e719) +- docs(release): the 10.4.0 entry, the index-health concept doc, and the API surfaces — written from the tree, not the plan (8cced871) +- fix(plugins): the silent-degrade doors close — a broken accelerator install can never read as absent (b9ba50fb) +- feat(recovery): the catchup verdict is consumed; verb rows go live; the metadata rebuild goes online (18f172e0) +- feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) + + ### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) - ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) diff --git a/package-lock.json b/package-lock.json index 8cee0038..b0120559 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.1", + "version": "10.4.0-rc.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.1", + "version": "10.4.0-rc.2", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index ef803c11..6bc4207c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.1", + "version": "10.4.0-rc.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From f4780c8e88c0e48acad704cff1ea78fe4aaef602 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 12:07:28 -0700 Subject: [PATCH 018/150] fix(update-seam): the metadata crossing never carries BigInt endpoint ints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveVerbEndpointInts mirrors the resolved u64 endpoint ints onto the verb object itself as BigInt (verb.sourceInt/targetInt) for the graph legs' own params. The live verb path's delete legs then reused that same object as the metadata-index crossing — and the seam's metadata is JSON-safe by contract (a native provider serializes it; u64 as Number corrupts above 2^53), so JSON.stringify threw and the whole transaction aborted. Found by the first joint pair gate; four downstream suites red from one crossing. The crossing now routes through a JSON-safe view that drops BigInt-valued top-level keys — endpoint ints ride their own op params on the graph legs, exactly as designed, and never the metadata crossing. Applied at the retraction helper (cascade + unrelate + transact mirrors) and updateRelation's remove leg. Pinned by driving the exact shape (relate resolves ints, remove cascades the same object) through a provider shim enforcing the JSON contract — red-proved against the unfixed path (the joint gate's verbatim error), green with the fix. --- src/brainy.ts | 36 ++++++++++++++++++-- tests/integration/verb-metadata-rows.test.ts | 36 ++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 59fbe20c..069a77c0 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -3831,13 +3831,45 @@ export class Brainy implements BrainyInterface { * @returns The operation to add to the caller's batch, or `null` when * nothing could be done (already narrated + tracked as degraded). */ + /** + * @description A JSON-safe view of a record bound for the metadata-index + * crossing. The seam's metadata is JSON-safe BY CONTRACT (a native provider + * serializes it; u64 ints as Number corrupt above 2^53) — but + * {@link resolveVerbEndpointInts} MIRRORS the resolved endpoint ints onto + * the verb object itself as BigInt (`verb.sourceInt`/`targetInt`), so a + * verb object reused as index metadata carried BigInts into + * JSON.stringify, which throws, aborting the whole transaction (found by + * the first joint pair gate). Endpoint ints ride their OWN op params on the + * graph legs — the metadata crossing drops every BigInt-valued top-level + * key instead of guessing at a lossy numeric encoding. + * @param metadata - The candidate index-metadata record. + * @returns The same object when already JSON-safe, else a shallow copy + * without the BigInt-valued keys. + */ + private static jsonSafeIndexMetadata(metadata: unknown): unknown { + if (metadata === null || typeof metadata !== 'object') return metadata + const rec = metadata as Record + let hasBigint = false + for (const k in rec) { + if (typeof rec[k] === 'bigint') { hasBigint = true; break } + } + if (!hasBigint) return metadata + const out: Record = {} + for (const k in rec) { + if (typeof rec[k] !== 'bigint') out[k] = rec[k] + } + return out + } + private metadataIndexRetractionOp( id: string, metadata: unknown, context: string ): Operation | null { if (metadata) { - return new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata, this.indexWriteGeneration) + return new RemoveFromMetadataIndexOperation( + this.metadataIndex, id, Brainy.jsonSafeIndexMetadata(metadata), this.indexWriteGeneration + ) } const prov = this.metadataIndex as unknown as { removeEntityById?: (id: string) => Promise @@ -5114,7 +5146,7 @@ export class Brainy implements BrainyInterface { // read above); `updatedMetadata` is the raw stored record just // persisted — the same shape relate()/rebuild() use to add. tx.addOperation( - new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, existing, this.indexWriteGeneration) + new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, Brainy.jsonSafeIndexMetadata(existing), this.indexWriteGeneration) ) tx.addOperation( new AddToMetadataIndexOperation(this.metadataIndex, params.id, updatedMetadata, this.indexWriteGeneration) diff --git a/tests/integration/verb-metadata-rows.test.ts b/tests/integration/verb-metadata-rows.test.ts index 79ce00d0..ff08132a 100644 --- a/tests/integration/verb-metadata-rows.test.ts +++ b/tests/integration/verb-metadata-rows.test.ts @@ -187,4 +187,40 @@ describe('verb metadata rows — the live path matches the rebuild walk', () => expect(await index.getIds('tag', 'parity-f')).toEqual([]) }) + + + it('the metadata crossing never carries BigInt endpoint ints — a cascade delete after graph resolution survives JSON', async () => { + // resolveVerbEndpointInts MIRRORS the resolved u64 ints onto the verb + // object as BigInt (verb.sourceInt/targetInt). A provider that JSON- + // serializes the metadata crossing dies on BigInt — found by the first + // joint pair gate. This pin drives the exact shape: relate (graph legs + // resolve ints), then remove the source entity (the cascade passes the + // SAME verb object to the retraction), through a provider shim that + // enforces the JSON-safety contract the way a native provider does. + const employee = await brain.add({ data: 'cascade employee', type: 'person' }) + const invoice = await brain.add({ data: 'cascade invoice', type: 'document' }) + await brain.relate({ from: employee, to: invoice, type: 'relatedTo' }) + const mgr: any = (brain as any).metadataIndex + const origRemove = mgr.removeFromIndex.bind(mgr) + const seen: unknown[] = [] + mgr.removeFromIndex = async (id: string, metadata?: unknown, generation?: bigint) => { + seen.push(metadata) + JSON.stringify(metadata) // the contract: throws on BigInt, exactly like a native crossing + return origRemove(id, metadata, generation) + } + try { + await brain.remove(employee) // cascades the relation's retraction + } finally { + mgr.removeFromIndex = origRemove + } + expect(seen.length).toBeGreaterThan(0) + for (const m of seen) { + if (m && typeof m === 'object') { + for (const [k, v] of Object.entries(m as Record)) { + expect(typeof v, `metadata key ${k} must be JSON-safe`).not.toBe('bigint') + } + } + } + }) + }) From bce2593e2496facbe94b0fdc53ca467f4b6063f9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 12:51:19 -0700 Subject: [PATCH 019/150] chore(release): 10.4.0-rc.3 --- CHANGELOG.md | 9 +++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4867f53..c2342d59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) + +- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) +- Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) +- fix(add): empty string is real data, not a missing field (258e9042) +- feat(vfs): implement readdir's recursive option — typed since 7.30, never read (fc516da6) +- feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) + + ### [10.4.0-rc.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) - test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) diff --git a/package-lock.json b/package-lock.json index b0120559..519bbfac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.2", + "version": "10.4.0-rc.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.2", + "version": "10.4.0-rc.3", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 6bc4207c..467b1fef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.2", + "version": "10.4.0-rc.3", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 9730835bdfe41586f367743e5b9f2fdc6b36666d Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 15:31:19 -0700 Subject: [PATCH 020/150] feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coverage denominator the health-by-accounting ratification named for the vector family — never built until now, and its absence was measured as the exact outage class it existed to prevent: a migrated store with canonical vectors and no derived index opened with the vector leg EMPTY, served [] from vector search with no error, and the report-driven read gate had nothing to refuse on (the provider's coverage invariant was honestly unledgered — the denominator was ours to supply). - getCanonicalCounts() gains vectors: { all } — the count of canonical nouns holding a REAL vector. Incremented where a vector lands (the isNew-gated metadata seam for explicit vectors — the same discipline that keeps HNSW neighbor-link re-saves from inflating counts; a narrow noteVectorLanded hook for the deferred-embed landing, gated on the worker's own pre-embed read). Decremented on a proven delete of a vectored noun; a vector-uncertain delete marks the ledger suspect rather than guessing (no new reads on the delete path). Recounted by the sanctioned recount; legacy counts.json derives it once (a deferred noun's vector file exists with an empty vector, so presence requires one content read at derivation — never on the hot path). - The open gate's vector leg: when a health-reporting provider claims serving while the index holds zero nodes and the ledger proves vectored canonical rows exist, open BUILDS (narrated) — routed through the provider's idempotent fillFromCanonical() when exposed (the joint door; a partial shortfall stays repair()'s operator business), the JS rebuild otherwise — or fails typed pre-serve. Scoped exactly: bare isReady() providers, migrating providers, and white-box size stubs open as before. Pinned end-to-end from the partner gate's probe shape (store with vectored canonical rows, no derived index, reopen → search serves N, never []), red-proved against the pre-fix path; the inverse (zero vectored rows) opens without building and serves [] honestly. --- src/brainy.ts | 145 +++++++++- src/coreTypes.ts | 53 +++- src/storage/adapters/baseStorageAdapter.ts | 48 +++- src/storage/adapters/fileSystemStorage.ts | 89 +++++++ src/storage/adapters/memoryStorage.ts | 11 + src/storage/baseStorage.ts | 73 +++++- .../operations/StorageOperations.ts | 35 ++- .../canonical-count-ledger.test.ts | 176 ++++++++++++- .../integration/vector-leg-open-build.test.ts | 247 ++++++++++++++++++ tests/lifecycle/biography.test.ts | 6 + 10 files changed, 851 insertions(+), 32 deletions(-) create mode 100644 tests/integration/vector-leg-open-build.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 069a77c0..543286a2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -2382,6 +2382,17 @@ export class Brainy implements BrainyInterface { [{ type: 'embed.landed', id, vector: newVector }], 'system:embed-landing' ) + // Vectored-noun ledger: the landing commit above carries a vector + // write with NO accompanying metadata operation, so the + // saveNounMetadata(..., hasVector) seam never fires for it — the + // narrow storage hook is the only seam left. `oldVector.length===0` + // (already known for free from the pre-embed read above) proves this + // is a GENUINE first landing, not a re-embed of an already-vectored + // row (e.g. a deferred update() on a row that already had a real + // vector) — the latter must never double-count. + if (oldVector.length === 0) { + await this.storage.noteVectorLanded?.(id) + } this.clearPendingEmbed(id) } catch (err) { prodLog.warn( @@ -3000,8 +3011,11 @@ export class Brainy implements BrainyInterface { const runInsert: TransactionFunction = async (tx) => { // Operation 1: Save metadata FIRST (TypeAwareStorage caching) // isNew=true: skip pre-read for rollback (entity doesn't exist yet) + // hasVector: the vectored-noun ledger counts this insert iff its + // vector is real/non-empty (never true for a deferred embed, whose + // stub `vector` is `[]` — it counts later, at landing). tx.addOperation( - new SaveNounMetadataOperation(this.storage, id, storageMetadata, true) + new SaveNounMetadataOperation(this.storage, id, storageMetadata, true, vector.length > 0) ) // Operation 2: Save vector data @@ -10568,7 +10582,9 @@ export class Brainy implements BrainyInterface { plan.postCommit.push(() => this.kickEmbedWorker()) } plan.operations.push( - new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew), + // hasVector: see the single-add() insert path's comment — never true + // for a deferred embed (stub vector `[]`; counted later at landing). + new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), ...(deferringEmbed ? [] @@ -16900,9 +16916,92 @@ export class Brainy implements BrainyInterface { const shouldRebuildMetadata = !metadataMigrating && (epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0)) - const shouldRebuildVector = + + // VECTOR LEG — the two-engine gate's last red: a migrated 7.x-era store + // can hold canonical vectored nouns with NO derived vector index built. + // `legNeedsRebuild`'s size-heuristic fallback (below) only fires off + // `hnswIndexSize === 0`, and its health-report branch trusts a + // provider's own `serving` verdict verbatim — but a provider's health + // report can legitimately say `serving: true` while vector coverage is + // honestly UNLEDGERED on ITS side too (an unledgered invariant never + // flips serving), so neither signal alone can tell "genuinely empty" + // apart from "never built". The canonical vectored-noun ledger + // (`getCanonicalCounts().vectors.all` — Deliverable 1) is the + // denominator that CAN tell them apart, and is compared here: + // - a CONFIDENT (non-suspect) ledger `> 0` while the reported node + // count is 0 is a proven coverage gap — force the build regardless + // of what a health report claims; + // - a CONFIDENT ledger `=== 0` while the node count is 0 proves there + // is nothing to load (e.g. every noun's embed is still deferred) — + // skip the size-heuristic fallback's blunt "always rebuild when + // empty" trigger, which otherwise wastes a full canonical walk for + // zero benefit on every cold open of such a store; + // - an unavailable/suspect ledger changes nothing — loud errors never + // quiet losses, so a doubtful ledger must never suppress a rebuild + // the old heuristic would have run. + // The bare `isReady()` boolean (no report, no `unledgered` concept) is + // NOT overridden — that signal is what fixed the 48-seconds-per-restart + // regression pinned in tests/unit/cold-open-rebuild-gate.test.ts (a + // disk-native provider legitimately reporting 0 resident while durable + // on disk), and re-deriving it from a denominator the provider itself + // has no way to consult would reopen exactly that regression. + const vectorAssessment = assessProviderHealth(this.index) + const vectorLedger = await this.storage.getCanonicalCounts?.() + const vectorLedgerAll = vectorLedger?.vectors.all + const vectorLedgerConfident = vectorLedger !== undefined && !vectorLedger.suspect + const vectorHasCoverageProof = vectorLedgerConfident && (vectorLedgerAll as number) > 0 + const vectorConfirmedEmpty = vectorLedgerConfident && vectorLedgerAll === 0 + + let vectorNeedsRebuild: boolean + if (vectorAssessment.via === 'is-ready') { + // Bare isReady() stays authoritative and UNMODIFIED — see above. + vectorNeedsRebuild = vectorAssessment.readiness === 'not-ready' + } else if (vectorAssessment.via === 'health-report') { + vectorNeedsRebuild = + vectorAssessment.readiness !== 'ready' || + (hnswIndexSize === 0 && vectorHasCoverageProof) + } else { + // size-heuristic / no provider (the built-in JS engine's own posture). + vectorNeedsRebuild = hnswIndexSize === 0 && !vectorConfirmedEmpty + } + + const shouldRebuildVector = !vectorMigrating && (epochStale || vectorNeedsRebuild) + + // Narration (and the FAIL-TYPED backstop below) are scoped EXACTLY to + // the defect this gate closes: a provider whose OWN health report + // claims `serving: true` — an affirmative "I am ready" a caller would + // otherwise trust outright — while the canonical ledger proves vector + // coverage is missing. This is deliberately NARROWER than "any branch + // where the ledger contributed to the decision": + // - the bare isReady() branch is untouched, as above (never in scope); + // - the health-report branch's OWN `readiness !== 'ready'` case is + // already an ordinary, PRE-EXISTING rebuild trigger (the provider + // admits not-ready) — not a ledger override, so not a "gap"; + // - the size-heuristic/no-provider branch's rebuild-when-empty is the + // SAME blunt trigger the code always had (`hnswIndexSize === 0`) + // — the ledger only ever SUPPRESSES a rebuild there (the confirmed- + // empty case), it never forces one the old heuristic wouldn't + // already have run. Marking that branch a "gap" too made the + // FAIL-TYPED backstop fire on ordinary white-box tests that stub + // rebuild() as a no-op and pin `size()` at 0 to drive OTHER + // assertions (e.g. migration-deference's isMigrating() coverage) — + // those are not silent-empty defects, so they must open exactly as + // before (tests/unit/brainy/migration-deference.test.ts). + const vectorCoverageGap = !vectorMigrating && - (epochStale || legNeedsRebuild(this.index, hnswIndexSize === 0)) + vectorAssessment.via === 'health-report' && + vectorAssessment.readiness === 'ready' && + hnswIndexSize === 0 && + vectorHasCoverageProof + if (vectorCoverageGap) { + prodLog.warn( + `[Brainy] open(): vector index reports ${hnswIndexSize} node(s) but the canonical ` + + `ledger holds ${vectorLedgerAll} vectored noun(s) — the derived vector index is ` + + `missing or unbuilt on this store. Forcing the vector rebuild rather than serving ` + + `silent-empty search results.` + ) + } + const shouldRebuildGraph = !graphMigrating && (epochStale || legNeedsRebuild(this.graphIndex, false)) @@ -16955,9 +17054,27 @@ export class Brainy implements BrainyInterface { // provider running its own background migration is skipped here (it owns // its index until it verifies-and-swaps). const rebuildStartTime = Date.now() + // The vector leg's build door, by contract with the native provider: a + // provider exposing fillFromCanonical() gets THAT call — idempotent, the + // provider's own init runs it first so this is the backstop — never a + // full rebuild() for a coverage gap. A PARTIAL shortfall deliberately + // triggers nothing here: that is repair()'s operator door. The JS index + // has no fill door and keeps its rebuild. + const vectorBuild = (): Promise => { + const fillDoor = (this.index as unknown as { fillFromCanonical?: () => Promise }) + .fillFromCanonical + if (vectorCoverageGap && typeof fillDoor === 'function') { + prodLog.warn( + `[Brainy] open(): vector coverage gap routes through the provider's ` + + `fillFromCanonical() (idempotent canonical fill), not a full rebuild.` + ) + return fillDoor.call(this.index) + } + return this.index.rebuild() + } await Promise.all([ shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(), - shouldRebuildVector ? this.index.rebuild() : Promise.resolve(), + shouldRebuildVector ? vectorBuild() : Promise.resolve(), shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve() ]) @@ -16998,6 +17115,24 @@ export class Brainy implements BrainyInterface { console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`) } + // Vector coverage verification: the coverage-gap rebuild above (see + // `vectorCoverageGap`) MUST have actually restored the ledger's + // vectored nouns. A provider that STILL reports 0 nodes after its own + // rebuild() ran — no JS (or provider) fallback could build from what's + // on disk — cannot silently complete open(): search would then serve + // empty results with no signal, exactly the defect this gate closes. + // FAIL TYPED, pre-serve, rather than let a broken vector leg pass as a + // successful open. + if (vectorCoverageGap && this.index.size() === 0) { + throw new VectorIndexNotReadyError( + `open(): the canonical ledger holds ${vectorLedgerAll} vectored noun(s) but the vector ` + + `index still reports 0 node(s) after rebuild() — the derived vector index could not ` + + `be restored from canonical. Refusing to serve silent-empty search results; ` + + `investigate the vector provider/storage, or repairIndex({ rebuild: ['vector'] }) ` + + `after restoring the underlying data.` + ) + } + // 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes // have now rebuilt and verified, so they match this build's epoch — // re-stamp the marker LAST, only here. A crash anywhere above leaves the diff --git a/src/coreTypes.ts b/src/coreTypes.ts index cf4a29e0..8112afd2 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -801,6 +801,17 @@ export interface DerivedFamilyDeclaration { export interface CanonicalCounts { nouns: { counted: number; all: number } verbs: { counted: number; all: number } + /** + * The count of canonical nouns holding a REAL (non-empty) vector — the + * coverage denominator a vector index's node-count ledger is measured + * against (`nodeCount === vectors.all` is the whole-store coverage + * verdict for the vector leg, the vector-side mirror of `nouns.all` for + * metadata/graph). A deferred-embed noun (`add({ deferEmbedding: true })`) + * counts only once its vector actually LANDS — its canonical record exists + * (counted in `nouns.all`) with an empty vector until then, so it is + * deliberately NOT counted here in the interim. + */ + vectors: { all: number } /** An unprovable delete has left the `all` scalars unverified since the last recount. */ suspect: boolean } @@ -819,14 +830,47 @@ export interface StorageAdapter { * Save noun metadata separately * @param id Noun ID * @param metadata Noun metadata + * @param hasVector - OPTIONAL vectored-noun ledger hint: `true` when this + * write is a FRESH insert (`isNew`) whose vector is a real, non-empty + * array — the caller already knows this for free (the insert's own + * `vector` local), so the increment rides the SAME isNew gate that + * already protects `totalNounCountAll` from double-counting on HNSW + * neighbor-link re-saves (`saveNoun_internal` re-runs on every link + * change; this metadata seam does not). Absent/`false` ⇒ no ledger + * action. A deferred-embed insert passes `false` (its vector lands + * later — see {@link StorageAdapter.noteVectorLanded}). */ - saveNounMetadata(id: string, metadata: NounMetadata): Promise + saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise /** * Delete noun metadata * @param id Noun ID + * @param priorRecord - OPTIONAL already-known metadata (the caller's + * pre-delete read) — see {@link StorageAdapter.deleteNoun}. + * @param hadVector - OPTIONAL vectored-noun ledger hint: `true`/`false` + * when the caller already knows (read as a side effect of ITS OWN delete + * flow — e.g. `remove()`'s pre-read for the vector-index removal — never + * a read added FOR this ledger), `undefined` when genuinely unknown. A + * known `true` decrements the vectored-noun ledger; a known `false` is a + * no-op (it was never counted); `undefined` marks the ledger SUSPECT + * rather than guessing — the delete path must never add a canonical read + * to answer this question. */ - deleteNounMetadata(id: string): Promise + deleteNounMetadata(id: string, priorRecord?: NounMetadata | null, hadVector?: boolean): Promise + + /** + * OPTIONAL narrow ledger hook: record that a canonical noun's vector just + * LANDED for the first time. Exists ONLY for the deferred-embedding + * lifecycle — the landing commit (`system:embed-landing`) carries a vector + * write with no accompanying metadata operation, so the normal + * `saveNounMetadata(..., hasVector)` seam never fires for it. Callers MUST + * call this only when the noun held NO real vector before this write (the + * deferred-embed worker already holds that fact for free, from its own + * pre-embed read — never an added read). A backend without vectored-noun + * tracking is a no-op via this method's absence (feature-detected). + * @param id - The noun whose vector just landed. + */ + noteVectorLanded?(id: string): Promise /** * Get noun with metadata combined @@ -875,8 +919,11 @@ export interface StorageAdapter { * REQUIRE re-reading the record being removed: when the internal read * returns `null` (replace race, or a ghost left by an earlier version) the * decrement falls back to this record instead of being silently skipped. + * @param hadVector OPTIONAL vectored-noun ledger hint — see + * {@link StorageAdapter.deleteNounMetadata}'s `hadVector` param, which + * this forwards to unchanged. */ - deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise + deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise /** * Save verb - Pure HNSW verb with core fields only diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 7c080b4c..22bf1366 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1041,12 +1041,27 @@ export abstract class BaseStorageAdapter implements StorageAdapter { */ protected totalNounCountAll = 0 protected totalVerbCountAll = 0 + /** + * The count of canonical nouns holding a REAL (non-empty) vector — the + * vector-side mirror of `totalNounCountAll` and the coverage denominator a + * vector index's node-count ledger is measured against. A deferred-embed + * noun (`add({ deferEmbedding: true })`) counts only once its vector + * LANDS (the `system:embed-landing` commit) — its canonical record exists + * (already counted in `totalNounCountAll`) with an empty vector until + * then. Maintained on the write path (a fresh insert whose vector is + * non-empty +1, a deferred embed's landing +1, a PROVEN delete of a + * vectored noun −1), persisted beside the other ALL scalars, recomputed by + * the sanctioned recount. Shares `allCountsSuspect` — no separate flag. + */ + protected totalVectoredNounCount = 0 /** * `true` when a delete could not prove whether the record existed (no * canonical read, no caller-provided prior) — the ALL scalar may be off by * the unprovable deletes since. Loud, persisted, and cleared only by the * sanctioned recount; a consumer reading the scalar as a ledger denominator - * must treat a suspect scalar as unverified, never as exact. + * must treat a suspect scalar as unverified, never as exact. Also covers + * `totalVectoredNounCount` — a delete whose vector-presence fact was + * unknowable marks this SAME flag rather than minting a second one. */ protected allCountsSuspect = false /** One narration per session for the suspect transition (never per delete). */ @@ -1083,15 +1098,19 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * The canonical count ledger — O(1), no I/O. `counted` is the user-facing * scalar (public/internal tiers, what `getNounCount()` returns); `all` is * the ALL-visibility scalar every unfiltered storage walk is measured - * against (the coverage-ledger denominator for derived-index providers). - * `suspect` is `true` when an unprovable delete has made `all` unverified - * since the last sanctioned recount (`rebuildTypeCounts`). - * @returns Both scalars per family plus the suspect flag. + * against (the coverage-ledger denominator for derived-index providers); + * `vectors.all` is the vectored-noun scalar — the coverage denominator for + * a vector index's node-count ledger specifically. + * `suspect` is `true` when an unprovable delete has made `all` (any + * family, including `vectors`) unverified since the last sanctioned + * recount (`rebuildTypeCounts`). + * @returns All scalars per family plus the suspect flag. */ async getCanonicalCounts(): Promise { return { nouns: { counted: this.totalNounCount, all: this.totalNounCountAll }, verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll }, + vectors: { all: this.totalVectoredNounCount }, suspect: this.allCountsSuspect } } @@ -1103,7 +1122,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * @param family - Which family's delete was unprovable. * @param id - The id whose existence could not be established. */ - protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void { + protected markAllCountsSuspect(family: 'noun' | 'verb' | 'noun-vector', id: string): void { this.allCountsSuspect = true if (!this.allCountsSuspectNarrated) { this.allCountsSuspectNarrated = true @@ -1116,6 +1135,23 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } } + /** + * OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorLanded}): + * record a deferred-embed noun's FIRST real vector landing. The caller + * (the deferred-embed worker) proves this is a genuine landing — not a + * re-embed of an already-vectored row — by observing its own pre-embed + * read's vector was empty, at no added storage cost. + * @param id - The noun whose vector just landed (retained for a future + * narration seam; the count itself needs no id-keyed state). + */ + async noteVectorLanded(id: string): Promise { + void id + this.totalVectoredNounCount++ + this.scheduleCountPersist().catch(() => { + // Ignore persist errors — the in-memory count is authoritative; a later op retries. + }) + } + /** * Increment count for entity type - O(1) operation. * Concurrency is handled by the process-global mutex diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 965e97b9..fd9dbb4c 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -2583,6 +2583,7 @@ export class FileSystemStorage extends BaseStorage { // record reads), persist, and never scan again. Absent keys are a // legacy file, not a zero — a zero here would make every provider's // coverage ledger read "over-posted" on a populated store. + let needsPersist = false if ( typeof counts.totalNounCountAll === 'number' && typeof counts.totalVerbCountAll === 'number' @@ -2601,6 +2602,29 @@ export class FileSystemStorage extends BaseStorage { `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + `every tier) and persisted; no further scan.` ) + needsPersist = true + } + + // The vectored-noun scalar (shipped after the ALL scalars above — a + // counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but + // still predate THIS key). Unlike the ALL scalars, presence cannot be + // decided from the id-directory listing alone: a deferred-embed + // noun's `vectors.json` EXISTS with an empty `vector: []` until its + // embed lands, so this derivation reads every noun's `vectors.json` + // ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost. + if (typeof counts.totalVectoredNounCount === 'number') { + this.totalVectoredNounCount = counts.totalVectoredNounCount + } else { + const vectored = await this.scanVectoredNounCount() + this.totalVectoredNounCount = vectored + console.warn( + `[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` + + `derived once by reading every noun's vectors.json (${vectored} vectored) and ` + + `persisted; no further scan.` + ) + needsPersist = true + } + if (needsPersist) { await this.persistCounts() } @@ -2643,6 +2667,12 @@ export class FileSystemStorage extends BaseStorage { this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false + // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT + // (a deferred-embed noun's file exists but holds an empty vector until + // its embed lands), so this is a full O(nouns) content scan — see + // scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here, + // alongside the rest of this from-disk recovery. + this.totalVectoredNounCount = await this.scanVectoredNounCount() // Sample some entities for the type distribution (don't read all). // Read the metadata files DIRECTLY with fs — this runs inside init(), @@ -2728,6 +2758,61 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs + * — the vector-side mirror of {@link readEntityMetadataRaw}, same + * reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`). + * @param entityDir - Absolute `entities/nouns//` directory. + * @returns The parsed vector record, or null when absent/unreadable. + */ + private async readEntityVectorRaw(entityDir: string): Promise { + const base = path.join(entityDir, 'vectors.json') + try { + return JSON.parse(await fs.promises.readFile(base, 'utf-8')) + } catch { + // fall through to the compressed variant + } + try { + const gz = await fs.promises.readFile(`${base}.gz`) + return JSON.parse(zlib.gunzipSync(gz).toString('utf-8')) + } catch { + return null + } + } + + /** + * Count canonical nouns holding a REAL (non-empty) vector — the vectored- + * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence + * cannot be decided from the id-directory listing alone: a deferred-embed + * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`) + * until its embed LANDS, so this walk reads every noun's `vectors.json` + * CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time + * legacy-counts.json derivation or a lost/corrupted counts.json recovery; + * the result is persisted so this scan never repeats. + */ + private async scanVectoredNounCount(): Promise { + const base = path.join(this.rootDir, 'entities', 'nouns') + let vectored = 0 + try { + const shards = await fs.promises.readdir(base, { withFileTypes: true }) + for (const shard of shards) { + if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue + const shardPath = path.join(base, shard.name) + const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) + for (const entry of ids) { + if (!entry.isDirectory()) continue + const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) + if (record && Array.isArray(record.vector) && record.vector.length > 0) { + vectored++ + } + } + } + } catch (error: any) { + if (error?.code !== 'ENOENT') throw error + } + return vectored + } + /** * Persist counts to filesystem storage */ @@ -2744,6 +2829,10 @@ export class FileSystemStorage extends BaseStorage { // written before the ledger existed; initializeCounts() derives them once. totalNounCountAll: this.totalNounCountAll, totalVerbCountAll: this.totalVerbCountAll, + // Vectored-noun ledger scalar — absent in files written before it + // existed; initializeCounts() derives it once (a content scan, see + // scanVectoredNounCount()'s JSDoc). + totalVectoredNounCount: this.totalVectoredNounCount, allCountsSuspect: this.allCountsSuspect, lastUpdated: new Date().toISOString() } diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index f55a5626..ab2a52d3 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -520,6 +520,12 @@ export class MemoryStorage extends BaseStorage { let totalNouns = 0 let totalVerbs = 0 + // Vectored-noun scalar: unlike the bare presence check above, this needs + // the vectors.json RECORD'S content — a deferred-embed noun's record + // exists with an empty `vector: []` until its embed lands. In-memory this + // is a free field access (no I/O), unlike the filesystem adapter's + // per-noun disk read. + let totalVectoredNouns = 0 // Scan all paths in objectStore for (const path of this.objectStore.keys()) { @@ -528,6 +534,10 @@ export class MemoryStorage extends BaseStorage { if (nounMatch) { // Type is in metadata, not path - just count total totalNouns++ + const record = this.objectStore.get(path) as { vector?: unknown } | undefined + if (Array.isArray(record?.vector) && record.vector.length > 0) { + totalVectoredNouns++ + } } // Count verbs (entities/verbs/{shard}/{id}/vectors.json) @@ -543,6 +553,7 @@ export class MemoryStorage extends BaseStorage { // A scan of every canonical record IS the ALL-visibility count. this.totalNounCountAll = totalNouns this.totalVerbCountAll = totalVerbs + this.totalVectoredNounCount = totalVectoredNouns this.allCountsSuspect = false } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 6f4c7f0e..d6ccc5fa 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1762,8 +1762,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete a noun from storage + * @param hadVector - OPTIONAL vectored-noun ledger hint, forwarded to + * {@link deleteNounMetadata} unchanged — see its JSDoc. */ - public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise { + public async deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise { await this.ensureInitialized() // FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the @@ -1780,7 +1782,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // LONGER wrapped in a blind catch that masked faults as "file didn't exist". // `priorMetadata` (the caller's pre-delete read) keeps the decrement honest // even when the canonical read inside returns null (replace race / ghost). - await this.deleteNounMetadata(id, priorMetadata) + await this.deleteNounMetadata(id, priorMetadata, hadVector) // Remove the now-empty entity container (a no-op for key/prefix stores). await this.removeCanonicalContainer(getNounVectorPath(id)) @@ -3392,11 +3394,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Save noun metadata to storage (now typed) * Routes to correct sharded location based on UUID + * @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc. */ - public async saveNounMetadata(id: string, metadata: NounMetadata): Promise { + public async saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise { // Validate noun type in metadata - storage boundary protection validateNounType(metadata.noun) - return this.saveNounMetadata_internal(id, metadata) + return this.saveNounMetadata_internal(id, metadata, hasVector) } /** @@ -3407,9 +3410,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { * This ensures counts are updated AFTER metadata exists, fixing the race condition * where storage adapters tried to read metadata before it was saved. * + * @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc. * @protected */ - protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise { + protected async saveNounMetadata_internal(id: string, metadata: NounMetadata, hasVector?: boolean): Promise { await this.ensureInitialized() // ID-first path - no type needed! @@ -3465,6 +3469,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { // record persists here so the ALL scalar never lags the tree. if (isNew) { this.totalNounCountAll++ + // Vectored-noun ledger: rides the SAME isNew gate (once per id, at + // creation) — this seam is metadata-write-driven and never re-runs on + // the HNSW neighbor-link re-saves that hit saveNoun_internal, so it + // cannot double-count. A deferred-embed insert passes hasVector=false + // (or omits it); its vector lands later via noteVectorLanded(). + if (hasVector) { + this.totalVectoredNounCount++ + } if (!(metadata.noun && isCounted)) { this.scheduleCountPersist().catch(() => { // Ignore persist errors — the in-memory count is authoritative; a later op retries. @@ -3860,8 +3872,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { * the skip permanently inflated the persisted totals (adds counted, paired * removals not decremented), and `Math.max(totalNounCount, scanned)` made * the inflation unfixable by any disk cleanup. + * @param hadVector - OPTIONAL vectored-noun ledger hint — see + * {@link StorageAdapter.deleteNounMetadata}'s JSDoc. This method never + * reads `vectors.json` to answer the question itself (a canonical read + * the delete path must never add); a caller that cannot supply the fact + * for free leaves it `undefined`, and the ledger goes SUSPECT rather + * than guessing. */ - public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise { + public async deleteNounMetadata( + id: string, + priorRecord?: NounMetadata | null, + hadVector?: boolean + ): Promise { await this.ensureInitialized() // Direct O(1) delete with ID-first path. Read the canonical record BEFORE @@ -3885,6 +3907,15 @@ export abstract class BaseStorage extends BaseStorageAdapter { } else { this.markAllCountsSuspect('noun', id) } + + // Vectored-noun ledger: a KNOWN vector fact decrements (or no-ops); + // an UNKNOWN one goes suspect rather than guessing — see @param hadVector. + if (hadVector === true) { + if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount-- + else this.markAllCountsSuspect('noun-vector', id) + } else if (hadVector === undefined) { + this.markAllCountsSuspect('noun-vector', id) + } this.scheduleCountPersist().catch(() => { // Ignore persist errors — the in-memory count is authoritative; a later op retries. }) @@ -4549,6 +4580,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { // construction. let allNouns = 0 let allVerbs = 0 + // Vectored-noun scalar: unlike `allNouns` (decided from the metadata.json + // LISTING alone), presence cannot be decided from the vectors.json + // listing alone — a deferred-embed noun's vectors.json EXISTS with an + // empty `vector: []` until its embed lands, so the file's CONTENT must be + // read. This walk already lists every path per shard (including + // vectors.json entries — `listCanonicalObjects` yields both legs), so + // reading them here costs one EXTRA read per noun beyond the metadata.json + // read above (doubling this walk's per-noun I/O) — honest cost, paid only + // by this diagnostic/repair recount, never on the hot path. + let allVectoredNouns = 0 // Scan noun shards for (let shard = 0; shard < 256; shard++) { @@ -4559,6 +4600,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { const paths = await this.listCanonicalObjects(shardDir) for (const path of paths) { + if (path.includes('/vectors.json')) { + try { + const vectorRecord = await this.readCanonicalObject(path) + if (vectorRecord && Array.isArray(vectorRecord.vector) && vectorRecord.vector.length > 0) { + allVectoredNouns++ + } + } catch (error) { + // Skip vector records that fail to load — best-effort ground truth, + // same as the metadata read below. + } + continue + } if (!path.includes('/metadata.json')) continue allNouns++ @@ -4634,17 +4687,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { // IS the proof an unprovable delete could not give. const nounsAllBefore = this.totalNounCountAll const verbsAllBefore = this.totalVerbCountAll + const vectoredBefore = this.totalVectoredNounCount this.totalNounCountAll = allNouns this.totalVerbCountAll = allVerbs + this.totalVectoredNounCount = allVectoredNouns this.allCountsSuspect = false this.countCache.clear() await this.persistCounts() prodLog.info( `[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` + - `ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` + - (nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs - ? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})` + `ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs / ${allVectoredNouns} vectored nouns` + + (nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs || vectoredBefore !== allVectoredNouns + ? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore} / ${vectoredBefore})` : ' (unchanged)') + ` — scalar + per-type persisted` ) diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index c1e9f1c1..8b2ebffe 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -52,7 +52,15 @@ export class SaveNounMetadataOperation implements Operation { private readonly storage: StorageAdapter, private readonly id: string, private readonly metadata: NounMetadata, - private readonly isNew: boolean = false + private readonly isNew: boolean = false, + /** + * OPTIONAL vectored-noun ledger hint: `true` when this write's paired + * vector (the SAME insert's `vector` local) is real/non-empty — see + * {@link StorageAdapter.saveNounMetadata}'s JSDoc for the isNew-gated, + * double-count-proof seam this rides. Default `false`: a deferred-embed + * insert (or any caller that doesn't know) never counts here. + */ + private readonly hasVector: boolean = false ) {} async execute(): Promise { @@ -62,7 +70,7 @@ export class SaveNounMetadataOperation implements Operation { : await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata') // Save new metadata - await this.storage.saveNounMetadata(this.id, this.metadata) + await this.storage.saveNounMetadata(this.id, this.metadata, this.hasVector) // Return rollback action return async () => { @@ -70,8 +78,10 @@ export class SaveNounMetadataOperation implements Operation { // Restore previous metadata await this.storage.saveNounMetadata(this.id, previousMetadata) } else { - // Delete newly created metadata - await this.storage.deleteNounMetadata(this.id) + // Delete newly created metadata. `this.hasVector` is the SAME fact + // this operation's own execute() used to (maybe) count the vectored + // ledger — reversing with it on rollback needs no new read. + await this.storage.deleteNounMetadata(this.id, undefined, this.hasVector) } } } @@ -140,7 +150,9 @@ export class SaveNounOperation implements Operation { // Note: Not all adapters implement deleteNoun // This is acceptable - metadata deletion makes entity invisible if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') { - await this.storage.deleteNoun(this.noun.id) + // `this.noun.vector` is the SAME record just written — the + // vectored-noun ledger fact is free (no added read) and exact. + await this.storage.deleteNoun(this.noun.id, undefined, this.noun.vector.length > 0) } } } @@ -198,14 +210,21 @@ export class DeleteNounMetadataOperation implements Operation { return async () => {} } + // Vectored-noun ledger fact: `previousNoun` is already read above for the + // before-image capture — no added read. `undefined` (noun genuinely + // absent, metadata-only ghost) is passed through honestly; the storage + // layer marks the ledger suspect rather than guessing. + const hadVector = previousNoun ? previousNoun.vector.length > 0 : undefined + // Full removal: both canonical legs + the entity container + count decrement // (the prior record keeps the decrement honest on a null canonical read). - await this.storage.deleteNoun(this.id, previousMetadata) + await this.storage.deleteNoun(this.id, previousMetadata, hadVector) // Return rollback action return async () => { // Restore the vector leg, then the metadata leg through the count-aware - // save so deleteNoun()'s decrement is reversed. + // save so deleteNoun()'s decrement is reversed (hadVector's mirror: + // re-increments the vectored ledger iff the restored vector is real). if (previousNoun) { await this.storage.saveNoun({ id: previousNoun.id, @@ -215,7 +234,7 @@ export class DeleteNounMetadataOperation implements Operation { }) } if (previousMetadata) { - await this.storage.saveNounMetadata(this.id, previousMetadata) + await this.storage.saveNounMetadata(this.id, previousMetadata, hadVector === true) } } } diff --git a/tests/integration/canonical-count-ledger.test.ts b/tests/integration/canonical-count-ledger.test.ts index 8d0f46b8..f3c8ce20 100644 --- a/tests/integration/canonical-count-ledger.test.ts +++ b/tests/integration/canonical-count-ledger.test.ts @@ -17,10 +17,11 @@ * (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is * upgraded from the canonical id tree at open, then persisted. */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' +import * as zlib from 'node:zlib' import { Brainy } from '../../src/index.js' /** Count canonical `/entities///` directories — every tier. */ @@ -180,3 +181,176 @@ describe('canonical count ledger — ALL-visibility scalars, unclamped totals, r expect(ledger.nouns.all).toBe(truth) }) }) + +/** Count `/entities/nouns///vectors.json[.gz]` files holding a non-empty `vector`. */ +function countVectoredNouns(root: string): number { + const base = path.join(root, 'entities', 'nouns') + if (!fs.existsSync(base)) return 0 + let n = 0 + for (const shard of fs.readdirSync(base)) { + const shardDir = path.join(base, shard) + if (!fs.statSync(shardDir).isDirectory()) continue + for (const id of fs.readdirSync(shardDir)) { + const idDir = path.join(shardDir, id) + if (!fs.statSync(idDir).isDirectory()) continue + const plainPath = path.join(idDir, 'vectors.json') + const gzPath = `${plainPath}.gz` + let record: any = null + if (fs.existsSync(plainPath)) { + record = JSON.parse(fs.readFileSync(plainPath, 'utf-8')) + } else if (fs.existsSync(gzPath)) { + record = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf-8')) + } else { + continue + } + if (Array.isArray(record.vector) && record.vector.length > 0) n++ + } + } + return n +} + +describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s coverage denominator)', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + /** Baseline vectored count right after a fresh open() — init() creates a + * hidden system VFS-root noun that itself carries a real vector, so a + * brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off + * this baseline rather than hardcoding it away. */ + let baseline: number + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-')) + brain = await open() + baseline = (await brain.storage.getCanonicalCounts()).vectors.all + }) + afterEach(async () => { + vi.restoreAllMocks() + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('an explicit-vector add counts immediately; the ledger matches the on-disk vectors.json content', async () => { + await brain.add({ data: 'a', type: 'document', vector: Array(384).fill(0).map((_, i) => Math.sin(i)) }) + await brain.add({ data: 'b', type: 'document' }) // embedded (non-deferred) — also a real vector + await brain.flush() + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(baseline + 2) + expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) + expect(ledger.suspect).toBe(false) + }) + + it('a deferred-embed add does NOT count until its embed LANDS', async () => { + // Hold the background worker's embed call open under manual control — a + // deterministic embedder is fast enough that the landing could otherwise + // race ahead of the "still unlanded" assertion below. + let resolveEmbed: ((v: number[]) => void) | undefined + vi.spyOn(brain, 'embed').mockImplementation( + () => new Promise((resolve) => { resolveEmbed = resolve }) + ) + + const id = await brain.add({ data: 'deferred content', type: 'document', deferEmbedding: true }) + await brain.flush() + + // Landed nothing yet — the ledger must not count the stub. + let ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(baseline) + expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) + + // Release the held embed, then cross the barrier: the vector lands + // (system:embed-landing). + resolveEmbed!(Array(384).fill(0).map((_, i) => Math.cos(i))) + await brain.awaitPendingEmbeds() + const landed = await brain.get(id, { includeVectors: true }) + expect((landed!.vector as number[]).length).toBeGreaterThan(0) + + ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(baseline + 1) + expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) + expect(ledger.suspect).toBe(false) + }) + + it('a proven delete of a vectored noun decrements; a non-vectored (unlanded) delete does not', async () => { + const vectoredId = await brain.add({ data: 'v', type: 'document' }) // real embed, unmocked + // Block the embed worker AFTER the real add above — a deterministic + // embedder is fast enough that the deferred noun below could otherwise + // land before this test observes its "still unlanded" state. + vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) + const deferredId = await brain.add({ data: 'd', type: 'document', deferEmbedding: true }) + await brain.flush() + expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(baseline + 1) + + await brain.remove(vectoredId) + await brain.flush() + let ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(baseline) + expect(ledger.suspect).toBe(false) + + await brain.remove(deferredId) // never had a real vector — no decrement, still unsuspect + await brain.flush() + ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(baseline) + expect(ledger.suspect).toBe(false) + expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) + }) + + it('the recount corrects a tampered vectors.all scalar, surviving reopen', async () => { + await brain.add({ data: 'real 1', type: 'document' }) + await brain.add({ data: 'real 2', type: 'document' }) + await brain.flush() + const truth = countVectoredNouns(dir) + expect(truth).toBe(baseline + 2) + + ;(brain.storage as any).totalVectoredNounCount = truth + 40 + await (brain.storage as any).persistCounts() + await brain.close() + brain = await open() + + // The lie survives reopen (never clamped). + expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth + 40) + + await brain.repairIndex() + expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth) + + await brain.close() + brain = await open() + expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth) + }) + + it('a legacy counts.json without totalVectoredNounCount is derived once from vectors.json content and persisted', async () => { + await brain.add({ data: 'one', type: 'document' }) // real embed, unmocked + // Block the embed worker AFTER the real add above — a deterministic + // embedder is fast enough that the deferred noun below could otherwise + // land before close(), which would inflate this test's expected count. + vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) + await brain.add({ data: 'two deferred', type: 'document', deferEmbedding: true }) + await brain.flush() + await brain.close() + + const countsPath = path.join(dir, '_system', 'counts.json') + const raw = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) + expect(typeof raw.totalVectoredNounCount).toBe('number') + delete raw.totalVectoredNounCount + fs.writeFileSync(countsPath, JSON.stringify(raw, null, 2)) + + brain = await open() + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun + expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) + const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) + expect(persisted.totalVectoredNounCount).toBe(baseline + 1) + }) +}) diff --git a/tests/integration/vector-leg-open-build.test.ts b/tests/integration/vector-leg-open-build.test.ts new file mode 100644 index 00000000..a1ccaefa --- /dev/null +++ b/tests/integration/vector-leg-open-build.test.ts @@ -0,0 +1,247 @@ +/** + * @module tests/integration/vector-leg-open-build + * @description THE LAST RED of the two-engine release gate: a migrated + * store can hold canonical vectored nouns with NO derived vector index + * built. `open()` owns building the derived indexes (reads never build — + * see `rebuildIndexesIfNeeded`'s JSDoc); the defect this pins is the vector + * leg's decision silently skipping that build, so `find`/search served `[]` + * with no error and no narration. + * + * A downstream deployment measures this through a native vector provider + * whose own health report can legitimately say `serving: true` even while + * vector COVERAGE is honestly unledgered on its side (an unledgered + * invariant never flips `serving` — see `HealthReport`'s derivation laws). + * This repo ships only the JS engine, so the reproduction here uses the + * SAME plugin seam a native provider would (`brain.use({ activate: ctx => + * ctx.registerProvider('vector', factory) })`, the pattern + * `tests/unit/cold-open-rebuild-gate.test.ts` already established for this + * exact class of gate-decision bug) with a stub that WRAPS the real + * `JsHnswVectorIndex` — every method delegates to a genuine engine (so a + * successful rebuild restores REAL, searchable vectors), except `size()` + * (fakes 0 until rebuild runs — the "never built" posture) and + * `healthReport()` (always reports `serving: true`, `unledgered: + * ['vector-coverage']` — the "I don't track this yet" posture). This is + * "as close as the JS engine allows": the gap is reproduced at the exact + * decision the fix changes, not approximated by deleting files the JS + * engine's own cold-start heuristic already recovers from unaided (see the + * inverse pin below and cold-open-rebuild-gate.test.ts's already-pinned + * "isReady()===true, size()===0" contract, which this fix deliberately does + * NOT touch — bare isReady() has no unledgered concept to hide behind, and + * overriding it would reopen the 48-seconds-per-restart regression pinned + * there). + * + * Pins: + * (1) COVERAGE GAP FORCES THE BUILD: N vectored nouns, a provider that + * claims `serving: true` at `size()===0` — open() builds anyway (the + * ledger proves there is something to cover), and search returns real + * results, never `[]`. + * (2) THE INVERSE, HONEST EMPTY: 0 vectored nouns (every embed still + * deferred/unlanded) — open() does NOT attempt a rebuild (nothing to + * load; the old blunt "always rebuild when size()===0" heuristic wasted + * a full canonical walk here for zero benefit), and search honestly + * returns `[]` — no error, no false coverage-gap narration. + * + * SEARCH VERIFICATION NOTE: pin (1) verifies "search returns real results" + * via `find({ query: })` (semantic search — embeds the query, then + * searches), matching the pattern `tests/integration/hnsw-rebuild.test.ts` + * already uses for exactly this "post-rebuild search works" class of pin. + * A raw `find({ vector: })` / `index.search(vector, k)` call was + * tried first and found to reproducibly return only 1 hit after a + * FROM-CANONICAL rebuild (never the full requested `limit`, sometimes not + * even a real neighbor) — REGARDLESS of this task's changes: it reproduces + * identically on a plain, unwrapped, un-stubbed reopen with the stock JS + * engine (verified against `hnsw-rebuild.test.ts`'s own construction) and + * is therefore a PRE-EXISTING, orthogonal defect in the JS HNSW engine's + * rebuilt-graph connectivity — outside this task's two deliverables (the + * count ledger and the open-gate REBUILD DECISION, not rebuild()'s internal + * search quality). Left for a separate investigation; not touched here. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vector-leg-open-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + vi.restoreAllMocks() + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +const V = (seed: number) => + Array.from({ length: 384 }, (_, i) => Math.sin((seed + 1) * 7919 + i * 131) * 0.5 + 0.5) + +/** + * Build a store with N explicit-vector (non-deferred) nouns, flush, close. + * Each noun also carries embeddable text (`technology`/`science`, matching + * the query used below) so the semantic-search verification exercises real + * retrieval, not a coincidental match. The default JS engine builds a fully + * current store — the epoch marker is stamped current at this open's + * completion, so a later reopen's `_indexEpochStale` is honestly false and + * cannot mask the ledger-gap decision under test (nothing here manufactures + * epoch drift). + */ +async function buildVectoredStore(dir: string, n: number): Promise { + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true, + dimensions: 384 + }) + await brain.init() + const ids: string[] = [] + for (let i = 0; i < n; i++) { + ids.push( + await brain.add({ + data: `doc ${i} about ${i % 2 === 0 ? 'technology' : 'science'}`, + type: 'document', + vector: V(i) + }) + ) + } + await brain.flush() + await brain.close() + return ids +} + +describe('vector-leg open-build (two-engine gate, last red)', () => { + it('coverage gap: a provider reporting serving:true at size()===0 is overridden by the vectored-noun ledger — open() builds, search returns real results', async () => { + const dir = mkTmp() + const ids = await buildVectoredStore(dir, 12) + + // Wrap the REAL JS engine so a successful rebuild restores genuine, + // searchable vectors — only `size()` and `healthReport()` are faked, + // simulating a native provider that has never built its own coverage of + // an unledgered invariant. + const calls = { rebuild: 0 } + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true, + dimensions: 384 + }) + brain.use({ + name: 'fake-native-vector-unledgered-coverage', + activate: async (ctx: any) => { + ctx.registerProvider('vector', (config: any, distance: any, options: any) => { + const real = new JsHnswVectorIndex(config, distance, options) + let rebuilt = false + const originalRebuild = real.rebuild.bind(real) + ;(real as any).rebuild = async (...args: any[]) => { + const r = await originalRebuild(...args) + calls.rebuild++ + rebuilt = true + return r + } + const originalSize = real.size.bind(real) + ;(real as any).size = () => (rebuilt ? originalSize() : 0) + ;(real as any).healthReport = () => ({ + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 0, + generation: 1, + unledgered: ['vector-coverage'] + }) + return real + }) + return true + } + }) + await brain.init() + + // WITHOUT any find() first: open() itself must have built the leg. + expect(calls.rebuild, 'open() forced the rebuild despite serving:true').toBe(1) + const status = await brain.getIndexStatus() + expect(status.hnswIndex.size).toBeGreaterThanOrEqual(ids.length) + + // Real, searchable results — never [] (see the module doc's SEARCH + // VERIFICATION NOTE for why this is a semantic `query`, not a raw + // `vector`, call). + const results = await brain.find({ query: 'technology document', limit: 5 }) + expect(results.length).toBeGreaterThan(0) + expect(results.length).not.toBe(0) + + await brain.close() + }) + + it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => { + // ARCHITECTURAL NOTE (found while building this pin): every brainy store + // carries ONE permanent, always-vectored noun beyond user data — the VFS + // root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, + // src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero + // (but non-empty, length-384) vector on EVERY store's first open — never + // deferred (a deliberate WASM-cold-compile-avoidance fix, see that + // file's comment) — and VFS init unconditionally re-creates it if + // missing, before the rebuild gate ever runs. A literal "0 vectored + // nouns" store is therefore unreachable through the public API; a + // brand-new store's `vectors.all` floor is 1, not 0. This pin verifies + // the law the task names in the ACHIEVABLE form: nouns whose embed is + // still deferred/unlanded contribute NOTHING to the vectored-noun ledger + // — the coverage-gap comparison sees exactly the root (1), never + // root+deferred — and semantic search over deferred-only user content + // honestly returns `[]` (no error, no false "coverage restored" claim). + const dir = mkTmp() + + const build: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true, + dimensions: 384 + }) + await build.init() + const rootOnlyLedger = await build.storage.getCanonicalCounts() + // Block the embedder permanently so every add below stays deferred and + // unlanded for the rest of this test (a fast deterministic embedder + // could otherwise land it before we ever observe the "still 0 extra" + // state). + vi.spyOn(build, 'embed').mockImplementation(() => new Promise(() => {})) + for (let i = 0; i < 5; i++) { + await build.add({ data: `deferred ${i}`, type: 'document', deferEmbedding: true }) + } + await build.flush() + const ledgerWithDeferred = await build.storage.getCanonicalCounts() + // The five deferred adds contributed ZERO to the vectored-noun ledger. + expect(ledgerWithDeferred.vectors.all).toBe(rootOnlyLedger.vectors.all) + await build.close() + + // Reopen (default JS engine — no stub needed): the root is the ONLY + // thing the vector leg has to load; the deferred nouns are correctly + // invisible to it. Block the embedder again BEFORE init() — reopen + // recovers the durable pending-embed markers and kicks the worker as + // part of init() itself, and an unblocked deterministic embedder could + // land all five before this test observes the open-time ledger. + const brain: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + plugins: [], + silent: true, + dimensions: 384 + }) + vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) + await brain.init() + // The ledger is exactly the root — the five deferred, still-unlanded + // nouns (which the rebuild above DOES insert into the graph, each with + // its stub empty vector — `index.size()` counts EVERY canonical noun's + // graph node, deferred or not, so it is not the coverage metric) never + // inflate the VECTORED count. + const ledgerAfterReopen = await brain.storage.getCanonicalCounts() + expect(ledgerAfterReopen.vectors.all).toBe(rootOnlyLedger.vectors.all) + + const results = await brain.find({ vector: V(3), limit: 5 }) + expect(results).toEqual([]) + + await brain.close() + }) +}) diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts index 9fda62a5..8b274fce 100644 --- a/tests/lifecycle/biography.test.ts +++ b/tests/lifecycle/biography.test.ts @@ -380,6 +380,12 @@ describe.sequential('lifecycle — the working store', () => { counted: aliveVerbs + model.vfsContainsVerbs, all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs }, + // Every noun this biography ever adds carries an explicit/computed + // vector (the harness never defers an embed), so the vectored-noun + // scalar tracks nouns.all exactly. + vectors: { + all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns + }, suspect: false }) } finally { From d3aacfaf244a079ca54e181c0b0a126031e97117 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 25 Aug 2026 16:00:16 -0700 Subject: [PATCH 021/150] chore(release): 10.4.0-rc.4 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2342d59..494ae950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) + +- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) + + ### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) - fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) diff --git a/package-lock.json b/package-lock.json index 519bbfac..9daf75f3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.3", + "version": "10.4.0-rc.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.3", + "version": "10.4.0-rc.4", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 467b1fef..fdee766f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.3", + "version": "10.4.0-rc.4", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 834149ed90722146e332a795e63e686e98a2c20c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 26 Aug 2026 08:03:48 -0700 Subject: [PATCH 022/150] =?UTF-8?q?docs(releases):=20the=2010.4.0=20entry?= =?UTF-8?q?=20catches=20up=20to=20the=20late=20trains=20=E2=80=94=20repair?= =?UTF-8?q?=20routing,=20the=20vector=20ledger=20and=20open-gate=20leg,=20?= =?UTF-8?q?the=20loud=20config=20guard,=20the=20JSON-safe=20crossing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 45420bd1..bc400412 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -105,6 +105,25 @@ the flag has no effect on timing. Full manual control still lives in writes to both, swaps atomically, and persists once after the swap. Reads never observe a partial index. `repairIndex({ rebuild: ['metadata'] })` uses it automatically. +- **Incremental heal is routed.** A provider invariant that asks for the + incremental heal (`heal: 'repair'`) now routes to the provider's own + `repair()` when it exposes one — re-posting exactly what its ledger names, + never a store-sized rebuild — and the post-heal re-read of the report decides + success; a repair that doesn't converge is recorded with the escalation named. +- **The vector family joins the count ledger.** `getCanonicalCounts()` gains + `vectors: { all }` — the count of canonical entities holding a real vector + (deferred-embed entities count when their vector lands). And the open gate + closes the vector leg: a store whose canonical rows hold vectors but whose + derived vector index is empty now builds at `open()` (or refuses with the + typed error) instead of silently serving empty vector-search results. +- **An unknown storage config shape fails loudly.** A nested `config` object + carrying a path-shaped key (a shape that was never supported) used to fall + through silently to the default shared directory — every instance writing one + store while callers believed each had its own. It now throws, naming the + canonical `path` key. +- **Relation index rows are JSON-safe.** Internal endpoint identifiers can no + longer ride the metadata-index crossing (a native provider serializes it); + they stay on the graph operations where they belong. - **A broken accelerator install can never read as "not installed."** The auto-detection free pass now requires the resolution error to name the accelerator package itself, exactly — a missing platform-binary sibling From efa042b52cd64187538984e5e17dfb3d83f47eae Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 26 Aug 2026 09:44:47 -0700 Subject: [PATCH 023/150] chore(release): 10.4.0 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 494ae950..a8d186da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) + +- docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) + + ### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) - feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) diff --git a/package-lock.json b/package-lock.json index 9daf75f3..4c85804b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.4", + "version": "10.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.4", + "version": "10.4.0", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index fdee766f..a8db7f0b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.4", + "version": "10.4.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 21e506e802804b7184f0d384098571e66d6c4b88 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 26 Aug 2026 10:19:20 -0700 Subject: [PATCH 024/150] =?UTF-8?q?docs(guide):=20the=20docs=20pipeline=20?= =?UTF-8?q?publishes=20through=20the=20ingest=20API=20=E2=80=94=20the=20se?= =?UTF-8?q?parate=20deploy=20step=20is=20retired?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c7336a18..6acd0e0d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -91,7 +91,7 @@ test: add/update tests (patch version bump) ## Docs Pipeline — soulcraft.com/docs -Docs in `docs/**/*.md` are published with the npm package (included in `files`) and synced to soulcraft.com/docs on every portal deploy. Frontmatter controls what appears publicly. +Docs in `docs/**/*.md` are published with the npm package (included in `files`) and go live on soulcraft.com/docs via the docs ingest API: the release script's `scripts/push-docs.js` step POSTs every public doc to `https://soulcraft.com/api/docs/ingest` (auth: `DOCS_INGEST_SECRET` in the environment). No separate deploy step is involved (the old deploy-to-publish flow was retired in a platform change, 2026-08). Frontmatter controls what appears publicly. ### Docs check triggers @@ -161,9 +161,9 @@ npm run release:major # Breaking changes (rare, manual decision) The script: verifies clean git state, builds, tests, bumps version, updates CHANGELOG.md, commits, tags, pushes, publishes to npm, and creates a GitHub release. After a successful release, remind the user: -> "Published. Deploy portal to pick up the new docs → go to the portal project and deploy." +> "Published. Docs are live on soulcraft.com/docs (pushed via the ingest API during the release) — spot-check a changed page with curl." -Do NOT deploy portal from here. Portal is always deployed separately from within the portal project. +There is no separate deploy step anymore. If the docs push failed (the script warns loudly), re-run `node scripts/push-docs.js` with `DOCS_INGEST_SECRET` set. ## Closed-Source Product Names — HARD RULE From c039411e08a71c74b19df62835417d718288e249 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 26 Aug 2026 13:55:11 -0700 Subject: [PATCH 025/150] fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two cures from the pair's first production adoption, both measured live. THE READ GATE IS PER-FAMILY. The report-driven gate refused on ANY provider's not-ready verdict at every read choke point — so a pure metadata find({ where }) was refused because the VECTOR leg was not serving, and a deployment's badge reads returned errors for a verdict that had nothing to do with them. A read may only be refused by the family it actually consults: metadata reads by the metadata leg (plus graph for a `connected` filter), vector search by the vector leg, traversal by the graph leg. Callers name what they need; the existing narration-once-per- generation and typed-refusal laws are unchanged within a family. NO RE-EMBED ON UNCHANGED DATA. update() — and its transact() planner — treated any write that carried `data` as a data change: with deferEmbedding it queued a landing, and the worker re-embedded and re-landed a vector for content that had not changed. A host heartbeat re-writing an unchanged row every few seconds therefore fed a live index-row loop on a production store. A write carrying the row's current data (structural compare, key order normalized) is now not a data change: no re-embed, no deferred landing, no vector rewrite; the metadata write itself still commits. A real change re-embeds exactly as before. Pinned in tests/integration/read-gate-scope-and-no-reembed.test.ts — both pins red-proved against the unfixed code with the production shapes verbatim. Two health-gate pins that encoded the old brain-global scope are re-pointed to the family their reads consult. --- src/brainy.ts | 69 +++++++++++++-- tests/integration/health-gate.test.ts | 13 +-- .../read-gate-scope-and-no-reembed.test.ts | 83 +++++++++++++++++++ 3 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 tests/integration/read-gate-scope-and-no-reembed.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 543286a2..6289e8f2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -3598,7 +3598,15 @@ export class Brainy implements BrainyInterface { // re-embed below — a stale vector left behind with no path to ever // correct itself (a quiet loss, not the deferred-but-eventually- // correct flicker the deferEmbedding contract promises). - const hasNewData = params.data !== undefined && params.data !== null + const rawHasNewData = params.data !== undefined && params.data !== null + // NO RE-EMBED ON UNCHANGED DATA: a write carrying the row's CURRENT data + // is not a data change — no re-embed, no deferred landing, no vector + // rewrite. A host heartbeat re-writing an unchanged row every few + // seconds fed a live index-row loop on a production store (each + // "change" landed a vector); the amplifier dies here regardless of how + // often the host writes. + const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) + const hasNewData = rawHasNewData && !dataUnchanged // MT5 deferred re-embedding: the OLD vector keeps serving semantic // search — stale-but-present, never absent (the flicker law) — until // the background worker embeds the new data and swaps it atomically. @@ -4209,7 +4217,7 @@ export class Brainy implements BrainyInterface { // writes while every non-find() read served empty from a not-ready // provider for 15 minutes. A CHECK only — it never builds; throws a typed // NotReady error if a provider's health report says it isn't serving. - this.ensureIndexesLoaded() + this.ensureIndexesLoaded(['graph']) const entityInt = this.graphEntityInt(uuid) if (entityInt === undefined) return [] const neighborInts = await this.graphIndex.getNeighbors(entityInt, options) @@ -6803,7 +6811,7 @@ export class Brainy implements BrainyInterface { // READ-SURFACE READINESS GATE (see filterIdsBelted): a CHECK only — it // never builds. open() already brought every provider to serving before // init() returned; this throws a typed NotReady error if one isn't. - this.ensureIndexesLoaded() + this.ensureIndexesLoaded(['metadata']) // Loudly flag a degraded derived index (failed init rebuild, or an // adopt-forward degraded commit) so a partial result is never mistaken for @@ -6815,6 +6823,13 @@ export class Brainy implements BrainyInterface { let params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query + // The vector and graph legs gate only the finds that consult them. + const consultsVector = Boolean( + (params.query && params.query.trim() !== '') || params.vector || params.near + ) + if (consultsVector) this.ensureIndexesLoaded(['vector']) + if (params.connected) this.ensureIndexesLoaded(['graph']) + // Id normalization (8.0): resolve the graph-traversal anchor id(s) so a // caller may constrain by natural key. Each maps to the canonical UUID // add() stored; real UUIDs pass through. Done once here so every downstream @@ -10663,7 +10678,10 @@ export class Brainy implements BrainyInterface { // content (see the identical hasNewData in update()); a plain truthy // check would silently skip re-embedding an emptied value and leave a // stale vector with no path to ever correct itself. - const hasNewData = params.data !== undefined && params.data !== null + const rawHasNewData = params.data !== undefined && params.data !== null + // No re-embed on unchanged data — the transact() mirror of update()'s rule. + const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) + const hasNewData = rawHasNewData && !dataUnchanged let vector = existing.vector if (params.vector) { if (this.dimensions && params.vector.length !== this.dimensions) { @@ -12158,7 +12176,7 @@ export class Brainy implements BrainyInterface { // writes while every non-find() read served empty from a not-ready // provider for 15 minutes. A CHECK only — it never builds; throws a typed // NotReady error if a provider's health report says it isn't serving. - this.ensureIndexesLoaded() + this.ensureIndexesLoaded(['metadata']) try { return await this.metadataIndex.getIdsForFilter(filter, opts) } catch (err) { @@ -14654,7 +14672,7 @@ export class Brainy implements BrainyInterface { // writes while every non-find() read served empty from a not-ready // provider for 15 minutes. A CHECK only — it never builds; throws a typed // NotReady error if a provider's health report says it isn't serving. - this.ensureIndexesLoaded() + this.ensureIndexesLoaded(['graph']) // 8.0 BigInt boundary: unmapped node → no relations. const nodeInt = this.graphEntityInt(nodeId) if (nodeInt === undefined) return [] @@ -16543,12 +16561,47 @@ export class Brainy implements BrainyInterface { * `prodLog.warn` ONCE per (provider, `report.generation`) — never once per * read — before any throw decision is made. */ - private ensureIndexesLoaded(): void { - const providers: ReadonlyArray BrainyError]> = [ + /** + * @description Whether two entity `data` payloads are the same content — + * the "no re-embed on unchanged data" comparison. Primitives compare by + * value; objects compare structurally with key order normalized. + * @param a - The incoming data. + * @param b - The stored data. + * @returns `true` when the content is identical. + */ + private static sameEntityData(a: unknown, b: unknown): boolean { + if (a === b) return true + if (a === null || b === null || typeof a !== typeof b) return false + if (typeof a !== 'object') return false + const stable = (v: unknown): string => + JSON.stringify(v, (_k, val) => + val && typeof val === 'object' && !Array.isArray(val) + ? Object.keys(val as Record).sort().reduce((o, k) => { + ;(o as Record)[k] = (val as Record)[k] + return o + }, {} as Record) + : val + ) + try { return stable(a) === stable(b) } catch { return false } + } + + private ensureIndexesLoaded( + families: ReadonlyArray<'vector' | 'metadata' | 'graph'> = ['vector', 'metadata', 'graph'] + ): void { + // PER-FAMILY SCOPE. This gate used to refuse on ANY provider's not-ready + // verdict at every read choke point — so a pure metadata find({where}) + // was refused because the VECTOR leg was not serving; a production + // deployment's badge reads returned 500s for exactly that reason on the + // pair's first adoption. A read may only be refused by the family it + // actually consults: metadata reads by the metadata leg (+ graph for a + // `connected` filter), vector search by the vector leg, traversal by the + // graph leg. Callers name what they need. + const all: ReadonlyArray BrainyError]> = [ ['vector', this.index, VectorIndexNotReadyError], ['metadata', this.metadataIndex, MetadataIndexNotReadyError], ['graph', this.graphIndex, GraphIndexNotReadyError] ] + const providers = all.filter(([name]) => families.includes(name)) for (const [name, provider, ErrorClass] of providers) { // Migration LOCK (#18) deference: a migrating provider owns its own diff --git a/tests/integration/health-gate.test.ts b/tests/integration/health-gate.test.ts index 4c4fb454..1c9a642d 100644 --- a/tests/integration/health-gate.test.ts +++ b/tests/integration/health-gate.test.ts @@ -186,6 +186,8 @@ describe('health gate (b) — unledgered is unknown: never blocks a serving prov }) describe('health gate (c) — degraded-but-serving narrates once per generation', () => { + // PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the + // degraded report lives on the family the read actually consults. it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => { const brain = new Brainy(createTestConfig({ silent: true })) await brain.init() @@ -195,7 +197,7 @@ describe('health gate (c) — degraded-but-serving narrates once per generation' const internals = internalsOf(brain) let generation = 1 - internals.index.healthReport = () => + internals.metadataIndex.healthReport = () => healthReport({ provider: 'vector', serving: true, @@ -216,7 +218,7 @@ describe('health gate (c) — degraded-but-serving narrates once per generation' await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) expect(countNarrations()).toBe(2) // generation bumped — a second narration - delete internals.index.healthReport + delete internals.metadataIndex.healthReport }) }) @@ -332,6 +334,7 @@ describe('health gate (f) — the ceremony door: explicit rebuild bypasses invar }) describe('health gate (g) — a throwing healthReport() is a contract violation, never read as healthy', () => { + // PER-FAMILY LAW (10.4.1): the throwing report sits on the family the read consults. it('healthReport() that throws refuses loudly with the typed NotReady error naming the throw', async () => { const brain = new Brainy(createTestConfig({ silent: true })) await brain.init() @@ -340,13 +343,13 @@ describe('health gate (g) — a throwing healthReport() is a contract violation, await brain.flush() const internals = internalsOf(brain) - internals.index.healthReport = () => { + internals.metadataIndex.healthReport = () => { throw new Error('accelerator: mmap window busy') } - await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(VectorIndexNotReadyError) + await expect(brain.find({ where: { team: 'atlas' } })).rejects.toBeInstanceOf(MetadataIndexNotReadyError) await expect(brain.find({ where: { team: 'atlas' } })).rejects.toThrow(/mmap window busy/) - delete internals.index.healthReport + delete internals.metadataIndex.healthReport }) }) diff --git a/tests/integration/read-gate-scope-and-no-reembed.test.ts b/tests/integration/read-gate-scope-and-no-reembed.test.ts new file mode 100644 index 00000000..b1319249 --- /dev/null +++ b/tests/integration/read-gate-scope-and-no-reembed.test.ts @@ -0,0 +1,83 @@ +/** + * @module tests/integration/read-gate-scope-and-no-reembed + * @description Two cures from the pair's first production adoption: + * (1) THE READ GATE IS PER-FAMILY — a not-serving VECTOR leg refuses vector + * search only; a pure metadata find({ where }) and graph traversal keep + * serving. The brain-global gate refused a deployment's badge reads for a + * vector-leg verdict that had nothing to do with them. + * (2) NO RE-EMBED ON UNCHANGED DATA — an update() carrying the row's current + * data lands no vector, defers no embed, rewrites nothing. A host + * heartbeat re-writing an unchanged row fed a live index-row loop. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, VectorIndexNotReadyError } from '../../src/index.js' + +describe('read gate scope + no re-embed on unchanged data', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-gate-scope-')) + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 }) + await brain.init() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('a not-serving VECTOR leg refuses vector search only — metadata and graph reads keep serving', async () => { + const a = await brain.add({ data: 'employee alpha', type: 'person', metadata: { status: 'active' } }) + const b = await brain.add({ data: 'employee beta', type: 'person', metadata: { status: 'active' } }) + await brain.relate({ from: a, to: b, type: 'relatedTo' }) + await brain.flush() + + // The vector provider says it is NOT serving (a rebuild-class failure). + brain.index.healthReport = () => ({ + provider: 'vector', healthy: false, serving: false, generation: 7, unledgered: [], + invariants: [{ name: 'node-coverage', holds: false, heal: 'rebuild', detail: 'posted 0 < canonical 2', source: 'ledger' }], + checkedAt: 1, durationMs: 1 + }) + try { + const byStatus = await brain.find({ where: { status: 'active' } }) + expect(byStatus.map((r: any) => r.id).sort(), 'metadata find serves').toEqual([a, b].sort()) + const rel = await brain.related(a) + expect(rel.length, 'graph traversal serves').toBe(1) + await expect(brain.find({ query: 'employee' }), 'vector search refuses typed').rejects.toBeInstanceOf(VectorIndexNotReadyError) + } finally { + delete brain.index.healthReport + } + }) + + it('update() with the row\'s current data re-embeds nothing; a real change re-embeds', async () => { + const id = await brain.add({ data: 'invoice 1042 pending', type: 'document', metadata: { n: 1 } }) + await brain.flush() + const before = (await brain.get(id, { includeVectors: true })).vector + const ledgerBefore = await brain.storage.getCanonicalCounts() + const logBefore = (await brain.transactionLog({ limit: 50 })).length + + // The heartbeat shape: same data, re-written, deferred. + for (let i = 0; i < 3; i++) { + await brain.update({ id, data: 'invoice 1042 pending', metadata: { n: 1, tick: i }, deferEmbedding: true }) + } + await brain.flush() + const after = (await brain.get(id, { includeVectors: true })).vector + const ledgerAfter = await brain.storage.getCanonicalCounts() + const log = await brain.transactionLog({ limit: 50 }) + expect(after, 'vector untouched by unchanged-data writes').toEqual(before) + expect(ledgerAfter.vectors.all, 'vectored ledger untouched').toBe(ledgerBefore.vectors.all) + expect(log.filter((e: any) => e.origin === 'system:embed-landing').length, 'no landing commit for unchanged data').toBe(0) + expect(log.length - logBefore, 'the metadata writes themselves still commit').toBe(3) + + // A REAL change re-embeds (deferred → the worker lands it). + await brain.update({ id, data: 'invoice 1042 PAID', deferEmbedding: true }) + await brain.flush() + const changed = (await brain.get(id, { includeVectors: true })).vector + expect(changed, 'a real data change re-embeds').not.toEqual(before) + expect((await brain.storage.getCanonicalCounts()).vectors.all, 'a re-embed of a vectored row never double-counts').toBe(ledgerBefore.vectors.all) + }) +}) From 7870dc4092619a65ed048f67d41a45a2fb3a4fbe Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 26 Aug 2026 14:29:20 -0700 Subject: [PATCH 026/150] chore(release): 10.4.1-rc.1 --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8d186da..dc8bc509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.1-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1-rc.1) (2026-08-26) + +- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) +- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) + + ### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) - docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) diff --git a/package-lock.json b/package-lock.json index 4c85804b..db490ee8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0", + "version": "10.4.1-rc.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.0", + "version": "10.4.1-rc.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index a8db7f0b..41b4aca3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0", + "version": "10.4.1-rc.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 2914e0eb42a43479245d40bab865a1886145dfc3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 26 Aug 2026 14:48:26 -0700 Subject: [PATCH 027/150] =?UTF-8?q?docs(concepts):=20the=20serving=20law?= =?UTF-8?q?=20=E2=80=94=20a=20failure=20is=20graded=20by=20whether=20an=20?= =?UTF-8?q?answer=20could=20be=20wrong,=20never=20by=20the=20cost=20of=20t?= =?UTF-8?q?he=20fix;=20reads=20refuse=20per=20family?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/concepts/index-health.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md index 18bf7010..96199f11 100644 --- a/docs/concepts/index-health.md +++ b/docs/concepts/index-health.md @@ -68,6 +68,19 @@ a maintenance window, a divergence `repairIndex()` will clean up on its own schedule. `serving: false` is not benign. It means this provider is refusing to answer, on its own word, right now. +**How a failure gets its grade — the serving law.** A provider grades `heal` by +one question only: *could an answer be wrong?* — never *how expensive is the +fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post +exactly what the ledger names, reads serving throughout); it can never withhold +serving just because healing it takes work. `serving` is withheld only by a +small, named set of rebuild-graded conditions — the index not initialized, its +durable state absent, a manifest naming files that are not resident, a replay +that did not complete cleanly — the states in which an answer could genuinely be +wrong. And a read is only ever refused by the family it actually consults: a +metadata filter is answered by the metadata index alone, vector search by the +vector index, traversal by the graph index — one family's refusal never blocks +another family's reads. + ## Reads refuse — they never rebuild A query that reaches a not-serving provider does not trigger a rebuild from inside From 0a19bbd8a76b3b532b17cd24257c11d88137ddac Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 26 Aug 2026 15:19:54 -0700 Subject: [PATCH 028/150] chore(release): 10.4.1 --- CHANGELOG.md | 6 ++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8d186da..ec925642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26) + +- fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) +- docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) + + ### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) - docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) diff --git a/package-lock.json b/package-lock.json index 4c85804b..70635835 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0", + "version": "10.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.0", + "version": "10.4.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index a8db7f0b..4125b073 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0", + "version": "10.4.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 8a5c1245a7e9a8c4c4a7307ef3f0b60355d76551 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 09:18:45 -0700 Subject: [PATCH 029/150] build: derive generated-file stamps from git commit time, not wall clock Two builds of the same source tree could publish different artifacts because buildEmbeddedPatterns.ts and buildTypeEmbeddings.ts stamped their generated output with new Date().toISOString(). Route both generators' "Generated:" header and the generatedAt runtime field through a shared resolver: newest git commit time among the generator's inputs (script + source data), falling back to the stamp already present in the previous output when git is unavailable (e.g. a published tarball build), and finally to a fixed epoch value. Every fallback logs to stderr so degradation is never silent. Regenerated both committed output files once so the tree carries deterministic stamps; no other content changed. --- scripts/buildEmbeddedPatterns.ts | 15 +++- scripts/buildTypeEmbeddings.ts | 18 +++- scripts/lib/deterministicStamp.ts | 118 +++++++++++++++++++++++++++ src/neural/embeddedPatterns.ts | 2 +- src/neural/embeddedTypeEmbeddings.ts | 4 +- 5 files changed, 148 insertions(+), 9 deletions(-) create mode 100644 scripts/lib/deterministicStamp.ts diff --git a/scripts/buildEmbeddedPatterns.ts b/scripts/buildEmbeddedPatterns.ts index 73e51224..c046df45 100644 --- a/scripts/buildEmbeddedPatterns.ts +++ b/scripts/buildEmbeddedPatterns.ts @@ -10,6 +10,7 @@ import { TransformerEmbedding } from '../src/utils/embedding.js' import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' +import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -97,13 +98,22 @@ async function buildEmbeddedPatterns() { // Convert to base64 for embedding in TypeScript const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - + + // Deterministic stamp: derived from the git commit time of this + // generator's inputs, never from wall-clock time — two builds of the + // same source tree must produce byte-identical output. + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') + const generatedStamp = resolveDeterministicStamp( + [path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath], + outputPath + ) + // Generate TypeScript file with everything embedded const tsContent = `/** * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${new Date().toISOString()} + * Generated: ${generatedStamp} * Patterns: ${libraryData.patterns.length} * Coverage: 94-98% of all queries * @@ -197,7 +207,6 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length} ` // Write the TypeScript file - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/buildTypeEmbeddings.ts b/scripts/buildTypeEmbeddings.ts index 61bcf238..688d6ac1 100644 --- a/scripts/buildTypeEmbeddings.ts +++ b/scripts/buildTypeEmbeddings.ts @@ -11,6 +11,7 @@ import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' import { NounType, VerbType } from '../src/types/graphTypes.js' +import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -373,12 +374,24 @@ async function buildTypeEmbeddings() { const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') + // Deterministic stamp: derived from the git commit time of this + // generator's inputs, never from wall-clock time — two builds of the + // same source tree must produce byte-identical output. + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') + const generatedStamp = resolveDeterministicStamp( + [ + path.join(__dirname, 'buildTypeEmbeddings.ts'), + path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts') + ], + outputPath + ) + // Generate TypeScript file const tsContent = `/** * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${new Date().toISOString()} + * Generated: ${generatedStamp} * Noun Types: ${nounTypes.length} * Verb Types: ${verbTypes.length} * @@ -395,7 +408,7 @@ export const TYPE_METADATA = { verbTypes: ${verbTypes.length}, totalTypes: ${totalTypes}, embeddingDimensions: ${embeddingDim}, - generatedAt: "${new Date().toISOString()}", + generatedAt: "${generatedStamp}", sizeBytes: { embeddings: ${buffer.byteLength}, base64: ${base64.length} @@ -494,7 +507,6 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n ` // Write the TypeScript file - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/lib/deterministicStamp.ts b/scripts/lib/deterministicStamp.ts new file mode 100644 index 00000000..c2a66604 --- /dev/null +++ b/scripts/lib/deterministicStamp.ts @@ -0,0 +1,118 @@ +/** + * Deterministic generation-stamp resolution for Brainy's build-time code + * generators. + * + * Two builds of the same source tree must produce byte-identical output. + * A wall-clock stamp (`new Date()`) breaks that guarantee, so every + * generator that writes a "Generated:" header or a `generatedAt` field + * into its output must resolve the stamp through this module instead. + * + * Resolution order: + * 1. The newest git commit timestamp among the generator's input files + * (the generator script itself always counts as an input). + * 2. If git metadata is unavailable (for example, building from a + * published npm tarball with no `.git` directory), the stamp already + * recorded in the previously generated output file. + * 3. If neither is available, the fixed epoch string + * `1970-01-01T00:00:00.000Z`. + * + * Every fallback logs a line to stderr — deterministic degradation is + * loud, never a silent divergence. + */ + +import { execFileSync } from 'child_process' +import * as fs from 'fs' + +const EPOCH_STAMP = '1970-01-01T00:00:00.000Z' +const STAMP_PATTERN = /\*\s*Generated:\s*(\S+)/ + +/** + * Resolve the deterministic stamp for a generator run. + * + * @param inputPaths Absolute paths to every file whose content determines + * the generator's output, including the generator script itself. + * @param previousOutputPath Absolute path to the previously generated + * file, used for the existing-stamp fallback when git is unavailable. + * @returns An ISO-8601 timestamp string that is deterministic for a given + * source tree. + */ +export function resolveDeterministicStamp( + inputPaths: string[], + previousOutputPath: string +): string { + const gitStamp = newestGitCommitTimestamp(inputPaths) + if (gitStamp) { + return gitStamp + } + + const existingStamp = readExistingStamp(previousOutputPath) + if (existingStamp) { + process.stderr.write( + `[deterministic-stamp] no git commit history found for generator inputs; ` + + `reusing existing stamp from ${previousOutputPath}: ${existingStamp}\n` + ) + return existingStamp + } + + process.stderr.write( + `[deterministic-stamp] no git commit history and no previous output at ` + + `${previousOutputPath}; falling back to fixed epoch stamp ${EPOCH_STAMP}\n` + ) + return EPOCH_STAMP +} + +/** + * Find the newest git commit timestamp among the given input paths. + * Returns null if git is unavailable, the tree is not a git repository, + * or none of the inputs have any commit history yet. + */ +function newestGitCommitTimestamp(inputPaths: string[]): string | null { + let newest: string | null = null + + for (const inputPath of inputPaths) { + if (!fs.existsSync(inputPath)) { + continue + } + + let out: string + try { + out = execFileSync( + 'git', + ['log', '-1', '--format=%cI', '--', inputPath], + { stdio: ['ignore', 'pipe', 'ignore'] } + ) + .toString() + .trim() + } catch { + // git missing, not a repository, or no permissions — handled by the + // caller's fallback chain. + continue + } + + if (!out) { + // Path exists but has no commit history yet (e.g. newly created, + // uncommitted file). + continue + } + + if (!newest || new Date(out).getTime() > new Date(newest).getTime()) { + newest = out + } + } + + return newest +} + +/** + * Parse the `* Generated: ` header out of a previously + * generated file, if one exists. + */ +function readExistingStamp(outputPath: string): string | null { + if (!fs.existsSync(outputPath)) { + return null + } + + const content = fs.readFileSync(outputPath, 'utf-8') + const match = content.match(STAMP_PATTERN) + return match ? match[1] : null +} diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index c15447e7..4f4339f4 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-07-02T21:43:26.976Z + * Generated: 2025-09-29T10:10:00-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index b5f3546b..5b10116c 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-02-09T16:59:48.867Z + * Generated: 2026-06-29T10:04:19-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-02-09T16:59:48.867Z", + generatedAt: "2026-06-29T10:04:19-07:00", sizeBytes: { embeddings: 259584, base64: 346112 From c6cc0de955e396ab3b65adc80ad92564cacbbf3e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 09:28:44 -0700 Subject: [PATCH 030/150] fix(vfs): the VFS root never persists a zero-norm vector A zero-norm vector is lawful inside brainy (cosine distance scores it at maximum, never a false top hit) but a false attractor for a downstream engine serving squared-euclidean distance, which cannot tell a real all-zero vector apart from a legitimate origin point. - The VFS root now persists with vector [] (the existing "unvectored" shape) instead of a real all-zero 384-dim placeholder, and is never routed into the deferred-embed pipeline. - A one-time migration in the root-init path detects a pre-fix store's all-zero placeholder root (by norm, not length) and rewrites it to [] through a new sanctioned Brainy method that keeps the canonical vectored-noun ledger honest and removes the row from the vector index. - The vector-index write seam (AddToVectorIndexOperation, ReplaceInVectorIndexOperation, and the generation materializer's direct insert) now refuses any real all-zero vector before it reaches a provider, loudly naming the entity, while the canonical write still lands. - add()'s dimension-pinning and HNSW-insert gates, and the add-params validator, now treat any empty vector as carrying no dimension information, closing a latent trap where an explicit `vector: []` would have pinned dimensions to 0. --- src/brainy.ts | 105 ++++++++- src/coreTypes.ts | 21 ++ src/storage/adapters/baseStorageAdapter.ts | 18 ++ src/transaction/operations/IndexOperations.ts | 55 +++++ src/utils/distance.ts | 23 ++ src/utils/paramValidation.ts | 10 +- src/vfs/VirtualFileSystem.ts | 100 ++++++--- .../canonical-count-ledger.test.ts | 12 +- .../integration/vector-leg-open-build.test.ts | 33 +-- tests/integration/vfs-root-zero-norm.test.ts | 203 ++++++++++++++++++ 10 files changed, 516 insertions(+), 64 deletions(-) create mode 100644 tests/integration/vfs-root-zero-norm.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 6289e8f2..b498e632 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -25,6 +25,7 @@ import { } from './storage/brainFormat.js' import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' +import { isZeroNormVector } from './utils/distance.js' import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, @@ -2913,7 +2914,13 @@ export class Brainy implements BrainyInterface { // Ensure dimensions are set (a deferred-embed stub carries no dimension // information — the worker's real vector goes through the same guard). - if (!deferringEmbed) { + // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose + // vector is the "unvectored" empty-array shape carries no dimension + // information, deferred or not — an explicit `vector: []` (e.g. the VFS + // root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must + // never pin `this.dimensions` to 0, which would poison every subsequent + // real embed's dimension check for the life of the store. + if (!deferringEmbed && vector.length > 0) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -3029,10 +3036,15 @@ export class Brainy implements BrainyInterface { }, true) ) - // Operation 3: Add to HNSW index (after entity saved). A deferred - // embed has nothing to index yet — the worker's atomic update - // inserts the real vector. - if (!deferringEmbed) { + // Operation 3: Add to HNSW index (after entity saved). Gated on + // `vector.length > 0`, not `!deferringEmbed`: a deferred embed has + // nothing to index yet (the worker's atomic update inserts the real + // vector later), and an explicit `vector: []` insert (the VFS root's + // zero-norm fix — permanently unvectored plumbing, never embedded) + // is exactly the same "nothing to index yet" shape. The zero-norm + // BELT (a real all-zero vector, non-empty) is enforced inside + // AddToVectorIndexOperation itself — see its JSDoc. + if (vector.length > 0) { tx.addOperation( new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration) ) @@ -10245,6 +10257,18 @@ export class Brainy implements BrainyInterface { for (const id of nounIds) { const noun = await snapshotStorage.getNoun(id) if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) { + // THE ZERO-NORM LAW: a direct provider-write seam (this materializer + // inserts one-by-one, bypassing AddToVectorIndexOperation's own + // belt) — apply the same refusal here rather than handing a false + // attractor to the ephemeral reader's index. + if (isZeroNormVector(noun.vector)) { + prodLog.warn( + `[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` + + `entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` + + `engine boundary (the materialized record is unaffected)` + ) + continue + } await reader.index.addItem({ id: noun.id, vector: noun.vector }) } } @@ -10519,7 +10543,10 @@ export class Brainy implements BrainyInterface { const vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) - if (!deferringEmbed) { + // Gated on `vector.length > 0` — see the single-add() insert path's + // matching comment: an explicit `vector: []` carries no dimension + // information either, deferred or not. + if (!deferringEmbed && vector.length > 0) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -10601,9 +10628,12 @@ export class Brainy implements BrainyInterface { // for a deferred embed (stub vector `[]`; counted later at landing). new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), - ...(deferringEmbed - ? [] - : [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]), + // Gated on `vector.length > 0` — see the single-add() insert path's + // matching comment: an explicit `vector: []` has nothing to index + // either, deferred or not. + ...(vector.length > 0 + ? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)] + : []), new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration) ) plan.touchedNouns.push(id) @@ -16032,6 +16062,63 @@ export class Brainy implements BrainyInterface { return !this.pluginRegistry.hasProvider('embeddings') } + /** + * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's + * persisted vector from a real (non-empty) vector to the "unvectored" + * empty-array shape: the vector record is rewritten to `[]`, the row is + * removed from the vector index (if present), and the vectored-noun + * ledger (`getCanonicalCounts().vectors.all`) is decremented through the + * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the + * coverage ledger never silently drifts. + * + * Exists SOLELY for the VFS root zero-norm migration (see + * `VirtualFileSystem.doInitializeRoot()`, which detects a persisted root + * whose vector is the legacy all-zero placeholder and calls this once per + * store). This is NOT a general-purpose "clear my vector" API — ordinary + * application data has no sanctioned path from vectored back to + * unvectored (`update()` refuses an empty vector as a dimension mismatch, + * by design). Never call this outside the VFS root migration. + * + * Idempotent: a noun already unvectored (`vector.length === 0`) or absent + * is a no-op — safe to call on every `init()`. + * + * @param id - The canonical noun id to migrate. + * @returns `true` if a migration write happened, `false` if the noun was + * already unvectored (or absent) — a no-op. + */ + async unvectorNounForRootMigration(id: string): Promise { + const noun = await this.storage.getNoun(id) + if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false + + await this.persistSingleOp({ nouns: [id] }, async (tx) => { + // Rewrite the vector leg to the unvectored shape. Placeholder adjacency + // (mirrors update()'s own SaveNounOperation staging) — the op preserves + // stored graph state when `connections.size === 0`. + tx.addOperation( + new SaveNounOperation(this.storage, { + id, + vector: [], + connections: new Map(), + level: 0 + }) + ) + // Remove from the vector index — safe even if the row was never + // actually indexed (RemoveFromVectorIndexOperation's removeItem is a + // no-op when the id is absent). + tx.addOperation( + new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration) + ) + }) + + // Vectored-noun ledger: this migration carries a vector write with no + // accompanying metadata operation (metadata is untouched), so the + // saveNounMetadata(..., hasVector) seam never fires for it — mirrors the + // deferred-embed LANDING path's use of the narrow storage hook, in + // reverse. + await this.storage.noteVectorUnlanded?.(id) + return true + } + /** * Setup embedder */ diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 8112afd2..4b018e94 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -872,6 +872,27 @@ export interface StorageAdapter { */ noteVectorLanded?(id: string): Promise + /** + * OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}: + * record that a canonical noun's vector was just REMOVED — rewritten from + * a real (non-empty) vector to the "unvectored" empty-array shape. Exists + * for the ONE sanctioned reverse migration this engine supports: the VFS + * root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and + * `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix + * store's all-zero placeholder root vector to `[]` and must decrement + * `vectors.all` through this hook so the coverage ledger never drifts. + * NOT a general-purpose "I removed a vector" callback — ordinary + * application data has no sanctioned path from vectored back to + * unvectored (`update()` refuses an empty vector as a dimension + * mismatch by design). Callers MUST call this only when the noun held a + * REAL vector immediately before this write (the caller already holds + * that fact for free, from its own pre-write read — never an added read). + * A backend without vectored-noun tracking is a no-op via this method's + * absence (feature-detected). + * @param id - The noun whose vector was just removed. + */ + noteVectorUnlanded?(id: string): Promise + /** * Get noun with metadata combined * @returns Combined HNSWNounWithMetadata or null diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 22bf1366..88018a66 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1152,6 +1152,24 @@ export abstract class BaseStorageAdapter implements StorageAdapter { }) } + /** + * OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}): + * the mirror of {@link noteVectorLanded} — record a noun's vector was just + * REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a + * caller that (incorrectly) fires this for a noun already unvectored would + * otherwise drive the ledger negative — clamped defensively, matching the + * delete path's `if (this.totalVectoredNounCount > 0)` guard. + * @param id - The noun whose vector was just removed (retained for a + * future narration seam; the count itself needs no id-keyed state). + */ + async noteVectorUnlanded(id: string): Promise { + void id + if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount-- + this.scheduleCountPersist().catch(() => { + // Ignore persist errors — the in-memory count is authoritative; a later op retries. + }) + } + /** * Increment count for entity type - O(1) operation. * Concurrency is handled by the process-global mutex diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 139c67fe..cfee5074 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -13,6 +13,8 @@ import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' +import { isZeroNormVector } from '../../utils/distance.js' +import { prodLog } from '../../utils/logger.js' /** * Backend identity stamped into an operation's emitted `name` string (e.g. @@ -88,6 +90,30 @@ export class AddToVectorIndexOperation implements Operation { } async execute(): Promise { + // THE ZERO-NORM LAW (the live provider-write seam's belt): a zero-norm + // vector is not a vector — it never crosses an engine boundary. This + // engine's own cosine distance treats an all-zero vector safely (a + // zero-norm operand always scores MAXIMUM distance, see + // {@link isZeroNormVector}'s JSDoc), but a downstream engine serving + // squared-euclidean distance cannot tell it apart from a legitimate + // origin point — a false attractor that silently darkens real results. + // The canonical write already landed (SaveNoun/SaveNounMetadata + // operations are staged ahead of this one in every caller) — only the + // INDEX INSERT is refused here, loudly, never a throw. A length-0 + // vector is the unrelated "unvectored" shape and is skipped silently + // (the same contract callers already rely on for deferred embeds). + if (this.vector.length === 0) { + return async () => {} + } + if (isZeroNormVector(this.vector)) { + prodLog.warn( + `[vector-index] refusing to index a zero-norm vector for entity ${this.id} — ` + + `a zero-norm vector is not a vector and never crosses an engine boundary ` + + `(the canonical write is unaffected; only the vector-index insert is skipped)` + ) + return async () => {} + } + // Check if item already exists (for rollback decision) const existed = await this.itemExists(this.id) @@ -263,6 +289,35 @@ export class ReplaceInVectorIndexOperation implements Operation { // One commit generation for the whole replace (both branches + rollback). const generation = this.generationFn?.() + // THE ZERO-NORM LAW (see AddToVectorIndexOperation's matching JSDoc): a + // real all-zero replacement vector must never land in the index — refuse + // loudly, canonical write unaffected. The row must not be left stale + // either: if it was genuinely indexed under `oldVector`, remove it + // rather than pretend the old vector still describes the row. A + // length-0 `newVector` (the unrelated "unvectored" shape) is handled the + // same way, silently — no caller today reaches this with an empty + // replacement (update() rejects a dimension-mismatched empty vector), + // but the seam stays consistent in case one ever legitimately does. + if (isZeroNormVector(this.newVector) || this.newVector.length === 0) { + const wasIndexed = this.oldVector.length > 0 && !isZeroNormVector(this.oldVector) + if (isZeroNormVector(this.newVector)) { + prodLog.warn( + `[vector-index] refusing to replace with a zero-norm vector for entity ${this.id} — ` + + `a zero-norm vector is not a vector and never crosses an engine boundary ` + + `(the canonical write is unaffected; the row is removed from the vector index instead)` + ) + } + if (wasIndexed) { + await this.index.removeItem(this.id, generation) + } + return async () => { + // Restore the declared before-state. + if (wasIndexed) { + await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + } + } + } + if (typeof index.updateItem === 'function') { // Atomic path: one in-place call, the row never leaves the index. await index.updateItem({ id: this.id, vector: this.newVector }, generation) diff --git a/src/utils/distance.ts b/src/utils/distance.ts index 36e9e8e5..d61bc12e 100644 --- a/src/utils/distance.ts +++ b/src/utils/distance.ts @@ -65,6 +65,29 @@ export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number = return 1 - similarity } +/** + * True when `vector` is a REAL (non-empty) all-zero vector — the "false + * attractor" shape this engine's own cosine distance treats safely (a + * zero-norm operand always scores the MAXIMUM distance, see + * {@link cosineDistance}) but a downstream engine serving squared-euclidean + * distance cannot distinguish from a legitimate origin point. THE LAW: a + * zero-norm vector is not a vector — it never crosses an engine boundary + * (never handed to a vector-index provider as a searchable item). + * + * A length-0 vector is the UNRELATED "unvectored, not yet embedded" shape + * (the deferred-embed stub, a permanently-vectorless system row) and is + * deliberately NOT zero-norm here — callers checking for "nothing to index" + * should test `vector.length === 0` separately; this only flags the + * dangerous non-empty all-zero case. + */ +export function isZeroNormVector(vector: readonly number[]): boolean { + if (vector.length === 0) return false + for (let i = 0; i < vector.length; i++) { + if (vector[i] !== 0) return false + } + return true +} + /** * Calculates the Manhattan (L1) distance between two vectors. * Lower values indicate higher similarity. diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index d43559dc..606516ac 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -588,8 +588,14 @@ export function validateAddParams(params: AddParams): void { ) } - // Validate vector dimensions if provided - if (params.vector) { + // Validate vector dimensions if provided. A length-0 vector is the + // "unvectored" shape — an explicit `vector: []` (e.g. the VFS root's + // permanently-vectorless creation, see + // VirtualFileSystem.doInitializeRoot()'s zero-norm fix) carries no + // dimension information, exactly like an absent vector or a deferred + // embed's internal stub, so it is exempt from the dimension check rather + // than refused as a "0-dimensional vector". + if (params.vector && params.vector.length > 0) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 90018863..59a16be4 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -11,6 +11,7 @@ import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' import { Entity, AddParams, RelateParams, FindParams, Relation } from '../types/brainy.types.js' import { NounType, VerbType } from '../types/graphTypes.js' +import { isZeroNormVector } from '../utils/distance.js' import { PathResolver } from './PathResolver.js' import { mimeDetector } from './MimeTypeDetector.js' import { @@ -89,16 +90,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Uses deterministic UUID format for storage compatibility private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' - // OPEN-PATH FIX: the dimension of the placeholder vector given to the VFS - // root when it is first created (see `doInitializeRoot`). Mirrors the - // built-in WASM embedding engine's fixed, hardcoded output size - // (all-MiniLM-L6-v2 — see src/embeddings/candle-wasm/src/lib.rs - // HIDDEN_SIZE and src/embeddings/EmbeddingManager.ts). Deliberately NOT - // derived from `brain.dimensions` — this constant only applies to the - // default-embedder branch, where the true output dimension is this fixed - // value by construction, never a moving target. - private static readonly VFS_ROOT_VECTOR_DIMENSIONS = 384 - /** * Construct a VFS bound to a Brainy instance. * @@ -241,9 +232,11 @@ export class VirtualFileSystem implements IVirtualFileSystem { private async doInitializeRoot(): Promise { const rootId = VirtualFileSystem.VFS_ROOT_ID - // Try to get existing root by fixed ID (O(1) lookup, not query) + // Try to get existing root by fixed ID (O(1) lookup, not query). + // includeVectors: true — the zero-norm migration below (leg 2) needs to + // inspect the persisted vector to detect the legacy placeholder shape. try { - const existingRoot = await this.brain.get(rootId) + const existingRoot = await this.brain.get(rootId, { includeVectors: true }) if (existingRoot) { // Root exists - verify metadata is correct @@ -260,6 +253,34 @@ export class VirtualFileSystem implements IVirtualFileSystem { }) } + // ZERO-NORM ROOT MIGRATION (one-time): a pre-fix store persisted the + // root with a REAL all-zero placeholder vector — lawful inside + // brainy (cosineDistance treats a zero-norm operand as MAXIMUM + // distance, see src/utils/distance.ts) but a "false attractor" for a + // downstream engine serving squared-euclidean distance, which cannot + // tell an all-zero vector apart from a legitimate origin point (a + // production incident silently darkened 150+ rows in a partner + // engine's index this way). THE LAW: a zero-norm vector is not a + // vector — it never crosses an engine boundary. Detect the legacy + // shape via NORM, not length or dimension (any real all-zero vector + // qualifies, not just the historical 384-dim one), and rewrite it to + // the "unvectored" `[]` shape through the sanctioned migration path + // (Brainy.unvectorNounForRootMigration — see its JSDoc), which keeps + // `getCanonicalCounts().vectors.all` honest and removes the row from + // the vector index. Idempotent: a store already on the new shape + // (vector.length === 0) takes the false branch below on every + // subsequent init() — a permanent no-op, not a one-time flag. + const existingVector = existingRoot.vector ?? [] + if (existingVector.length > 0 && isZeroNormVector(existingVector)) { + const migrated = await this.brain.unvectorNounForRootMigration(rootId) + if (migrated) { + console.log( + 'VFS: migrated root vector from the legacy all-zero placeholder to the ' + + 'unvectored shape (zero-norm vectors never cross an engine boundary)' + ) + } + } + return rootId } } catch (error) { @@ -277,31 +298,42 @@ export class VirtualFileSystem implements IVirtualFileSystem { // meant every writer's FIRST-EVER open forced the process-global WASM // engine to cold-compile its model (measured 90-140s on throttled // CPUs) before the brain could even finish init(). This branch only - // runs once per store (the root already exists — with a real vector — - // in every previously-opened production store; reopening never re-adds - // it), so the fix applies only to brand-new stores. + // runs once per store (a previously-opened store already has a root — + // see the migration above for the pre-fix shape — reopening never + // re-adds it), so the fix applies only to brand-new stores. // - // Chosen fix: an explicit all-zero vector, not `deferEmbedding: true`. - // `deferEmbedding` looked attractive (ack now, embed later) but its - // landing path (`kickEmbedWorker()`, called synchronously right after - // commit — see brainy.ts add()/update()) would still force the WASM - // engine to cold-compile within milliseconds of open, just off the - // awaited path instead of never paying it at all — worse than the - // explicit-vector path, which never touches the engine for this row. - // An all-zero vector is safe: `cosineDistance` (src/utils/distance.ts) - // explicitly returns the MAXIMUM distance whenever either operand's - // norm is zero, so the root can never rank ahead of real content in a - // similarity search, and HNSW indexes it like any other vector. + // ZERO-NORM LAW (current shape, superseding the historical all-zero + // placeholder): the root's vector is `[]` — the SAME "unvectored" + // empty-array shape used for a deferred embed's stub and any other + // not-yet-embedded row — never a real all-zero vector. A zero-norm + // vector is lawful inside brainy (`cosineDistance`, see + // src/utils/distance.ts, returns the MAXIMUM distance whenever either + // operand's norm is zero) but is a "false attractor" for a downstream + // engine serving squared-euclidean distance, which cannot tell a real + // all-zero vector apart from a legitimate origin point — it silently + // darkened 150+ rows in a partner engine's index in production. THE + // LAW: a zero-norm vector is not a vector — it never crosses an engine + // boundary. `vector: []` achieves the SAME cold-compile avoidance the + // original placeholder did (`add()`'s dimension-pin and HNSW-insert + // gates both key off `vector.length > 0`, so an empty vector never + // calls embed(), never pins `brain.dimensions`, and never reaches the + // vector index — see brainy.ts add()'s matching comments) while never + // persisting a searchable zero vector for a downstream engine to trip + // over. Deliberately NOT `deferEmbedding: true`: that flag's landing + // path (`kickEmbedWorker()`, called synchronously right after commit — + // see brainy.ts add()/update()) would still force the WASM engine to + // cold-compile within milliseconds of open (just off the awaited path + // instead of never paying it at all) AND would eventually embed the + // root's data for real, which this fix forbids — the root must NEVER + // be embedded, not merely "not yet". // - // Only the default WASM engine gets this treatment — its output - // dimension (384) is fixed and hardcoded, so the placeholder can never - // mis-pin `brain.dimensions` for it. A plugin-registered native - // 'embeddings' provider has no cold-compile cost AND may use a - // different dimension, so it keeps embedding the root for real (same - // as before this fix) rather than risk pinning the wrong dimension - // ahead of the caller's own first real embed. + // Only the default WASM engine gets this treatment — a plugin- + // registered native 'embeddings' provider has no cold-compile cost and + // may use a different dimension, so it keeps embedding the root for + // real (same as before this fix) rather than leave Brainy's own + // plumbing permanently unvectored on a store where embedding is cheap. const rootVector = this.brain.usesDefaultWasmEmbedder() - ? new Array(VirtualFileSystem.VFS_ROOT_VECTOR_DIMENSIONS).fill(0) + ? ([] as number[]) : undefined await this.brain.add({ diff --git a/tests/integration/canonical-count-ledger.test.ts b/tests/integration/canonical-count-ledger.test.ts index f3c8ce20..3d292e97 100644 --- a/tests/integration/canonical-count-ledger.test.ts +++ b/tests/integration/canonical-count-ledger.test.ts @@ -225,9 +225,12 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s } /** Baseline vectored count right after a fresh open() — init() creates a - * hidden system VFS-root noun that itself carries a real vector, so a - * brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off - * this baseline rather than hardcoding it away. */ + * hidden system VFS-root noun, but (the zero-norm root cure) it is + * deliberately UNVECTORED (`vector: []`, never a real all-zero + * placeholder — a zero-norm vector never crosses an engine boundary), so + * a brand-new store's `vectors.all` is 0. Tests still assert DELTAS off + * this baseline rather than hardcoding it away, in case that ever + * changes again. */ let baseline: number beforeEach(async () => { @@ -235,6 +238,7 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-')) brain = await open() baseline = (await brain.storage.getCanonicalCounts()).vectors.all + expect(baseline).toBe(0) // the unvectored VFS root contributes nothing }) afterEach(async () => { vi.restoreAllMocks() @@ -348,7 +352,7 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s brain = await open() const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun + expect(ledger.vectors.all).toBe(baseline + 1) // just the one non-deferred noun — the root is unvectored expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) expect(persisted.totalVectoredNounCount).toBe(baseline + 1) diff --git a/tests/integration/vector-leg-open-build.test.ts b/tests/integration/vector-leg-open-build.test.ts index a1ccaefa..ea841545 100644 --- a/tests/integration/vector-leg-open-build.test.ts +++ b/tests/integration/vector-leg-open-build.test.ts @@ -176,21 +176,22 @@ describe('vector-leg open-build (two-engine gate, last red)', () => { }) it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => { - // ARCHITECTURAL NOTE (found while building this pin): every brainy store - // carries ONE permanent, always-vectored noun beyond user data — the VFS - // root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, - // src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero - // (but non-empty, length-384) vector on EVERY store's first open — never - // deferred (a deliberate WASM-cold-compile-avoidance fix, see that - // file's comment) — and VFS init unconditionally re-creates it if - // missing, before the rebuild gate ever runs. A literal "0 vectored - // nouns" store is therefore unreachable through the public API; a - // brand-new store's `vectors.all` floor is 1, not 0. This pin verifies - // the law the task names in the ACHIEVABLE form: nouns whose embed is - // still deferred/unlanded contribute NOTHING to the vectored-noun ledger - // — the coverage-gap comparison sees exactly the root (1), never - // root+deferred — and semantic search over deferred-only user content - // honestly returns `[]` (no error, no false "coverage restored" claim). + // ARCHITECTURAL NOTE (updated by the zero-norm root cure): every brainy + // store carries ONE permanent VFS root noun beyond user data + // (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, + // src/vfs/VirtualFileSystem.ts), created (or, on a pre-fix store, + // migrated) on every open — but it is deliberately UNVECTORED (vector + // `[]`), never a real all-zero placeholder: a zero-norm vector is not a + // vector and never crosses an engine boundary (see that file's + // doInitializeRoot() comment). It therefore contributes NOTHING to the + // vectored-noun ledger — a brand-new store's `vectors.all` floor is 0, + // not 1. This pin verifies the law the task names in the ACHIEVABLE + // form: nouns whose embed is still deferred/unlanded contribute NOTHING + // to the vectored-noun ledger either — the coverage-gap comparison sees + // exactly the baseline (the root, contributing 0), never + // baseline+deferred — and semantic search over deferred-only user + // content honestly returns `[]` (no error, no false "coverage restored" + // claim). const dir = mkTmp() const build: any = new Brainy({ @@ -202,6 +203,8 @@ describe('vector-leg open-build (two-engine gate, last red)', () => { }) await build.init() const rootOnlyLedger = await build.storage.getCanonicalCounts() + // THE NEW LAW: the root is unvectored — a brand-new store's floor is 0. + expect(rootOnlyLedger.vectors.all).toBe(0) // Block the embedder permanently so every add below stays deferred and // unlanded for the rest of this test (a fast deterministic embedder // could otherwise land it before we ever observe the "still 0 extra" diff --git a/tests/integration/vfs-root-zero-norm.test.ts b/tests/integration/vfs-root-zero-norm.test.ts new file mode 100644 index 00000000..00260eb4 --- /dev/null +++ b/tests/integration/vfs-root-zero-norm.test.ts @@ -0,0 +1,203 @@ +/** + * @module tests/integration/vfs-root-zero-norm + * @description THE ZERO-NORM ROOT CURE — a production incident traced 150+ + * darkened rows in a downstream engine's index to the VFS root's persisted + * ALL-ZERO placeholder vector: lawful inside brainy (`cosineDistance` + * treats a zero-norm operand as MAXIMUM distance, src/utils/distance.ts) + * but a "false attractor" for an engine serving squared-euclidean distance, + * which cannot tell a real all-zero vector apart from a legitimate origin + * point. THE LAW: a zero-norm vector is not a vector — it never crosses an + * engine boundary. + * + * Three legs pinned here: + * (a) the root persists NO zeros — a brand-new store creates it with + * vector `[]` (the "unvectored" shape), absent from the HNSW index, and + * the canonical vectored-noun ledger does not count it. + * (b) a ONE-TIME migration heals an existing (pre-fix) store: an old-shape + * root (a REAL all-zero vector, genuinely indexed and ledgered — the + * harness reproduces exactly what a pre-fix store looked like on disk) + * is rewritten to `[]` on the next `init()`, the ledger is decremented + * through the sanctioned path, and a second `init()` is a no-op. + * (c) THE BELT at the live provider-write seam: an entity added with an + * EXPLICIT all-zero vector (any dimension) still lands its canonical + * write, but the vector-index insert is refused loudly. + * (d) the migrated root never surfaces in `find()` results (it was already + * hidden behind `visibility: 'system'` — this pin holds regardless). + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { prodLog } from '../../src/utils/logger.js' + +const ROOT_ID = '00000000-0000-0000-0000-000000000000' + +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vfs-root-zero-norm-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + vi.restoreAllMocks() + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) +} + +describe('VFS root zero-norm cure', () => { + it('(a) a brand-new store persists the root with vector [], absent from the HNSW index, and the canonical ledger counts it unvectored', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const root = await brain.get(ROOT_ID, { includeVectors: true }) + expect(root).not.toBeNull() + expect(root.vector).toEqual([]) + + const status = await brain.getIndexStatus() + expect(status.hnswIndex.size).toBe(0) + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(0) + + await brain.close() + }) + + it('(b) an old-shape store (a real all-zero placeholder root) migrates to [] exactly once on init; the ledger is decremented through the sanctioned path; a second init is a no-op', async () => { + const dir = mkTmp() + + // SESSION 1 — build the store, then hand-rewrite the root to the LEGACY + // shape: a REAL all-zero 384-dim vector, genuinely inserted into the + // vector index and genuinely counted by the vectored-noun ledger — + // reproducing exactly what a pre-fix store's root looked like on disk + // (the pre-fix add() always indexed + counted it). `index.addItem` is + // called directly (bypassing AddToVectorIndexOperation's own zero-norm + // belt, added by this same fix) precisely because the pre-fix code path + // had no such belt — this harness must match history, not the cure. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.flush() + + const ledgerBeforeMigration = await brain.storage.getCanonicalCounts() + expect(ledgerBeforeMigration.vectors.all).toBe(1) + await brain.close() + + // SESSION 2 — reopen: VFS init must detect the legacy shape and migrate. + // Spy on the sanctioned migration method itself (not console output — + // `silent: true` monkey-patches `console.log` to a no-op INSIDE init(), + // which would silently swallow any pre-installed console spy too). + brain = openBrain(dir) + const migrateSpy = vi.spyOn(brain, 'unvectorNounForRootMigration') + await brain.init() + + expect(migrateSpy).toHaveBeenCalledTimes(1) + expect(migrateSpy).toHaveBeenCalledWith(ROOT_ID) + await expect(migrateSpy.mock.results[0].value).resolves.toBe(true) + + const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) + expect(migratedRoot.vector).toEqual([]) + + const ledgerAfterMigration = await brain.storage.getCanonicalCounts() + expect(ledgerAfterMigration.vectors.all).toBe(0) + + const statusAfterMigration = await brain.getIndexStatus() + expect(statusAfterMigration.hnswIndex.size).toBe(0) + + await brain.flush() + await brain.close() + + // SESSION 3 — reopen again: the migration is a permanent no-op, not a + // one-time flag that silently re-drifts or re-fires. The zero-norm + // detection at the VFS init site never even calls the migration method + // again — the root's vector is already `[]`. + brain = openBrain(dir) + const migrateSpy2 = vi.spyOn(brain, 'unvectorNounForRootMigration') + await brain.init() + + expect(migrateSpy2).not.toHaveBeenCalled() + + const rootAfterSecondInit = await brain.get(ROOT_ID, { includeVectors: true }) + expect(rootAfterSecondInit.vector).toEqual([]) + + const ledgerAfterSecondInit = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecondInit.vectors.all).toBe(0) + + await brain.close() + }) + + it('(c) the live-write belt: an entity added with an explicit all-zero vector lands its canonical write, but the vector-index insert is refused loudly', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const warnSpy = vi.spyOn(prodLog, 'warn') + + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const zeroVector = new Array(384).fill(0) + const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) + + // The canonical write succeeded — the entity is fully readable with its + // (real, all-zero) vector intact. + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual(zeroVector) + + // The vector-index insert was skipped — the index size never moved. + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore) + + // The refusal was LOUD and named the entity. + const loudCall = warnSpy.mock.calls.find( + (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') + ) + expect(loudCall).toBeDefined() + + await brain.close() + }) + + it('(d) find() over a store whose root has been migrated never returns the root (already hidden behind visibility: system — pinned anyway)', async () => { + const dir = mkTmp() + + // Build an old-shape store (same harness as pin (b)) and let it migrate. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.add({ data: 'a document about technology', type: NounType.Document }) + await brain.flush() + await brain.close() + + brain = openBrain(dir) // migrates on init() + await brain.init() + + const results = await brain.find({ query: 'technology', limit: 10 }) + expect(results.some((r: any) => r.id === ROOT_ID)).toBe(false) + + // Even asking explicitly for system-tier entities must never surface the + // root as a semantic-search HIT (it carries no vector to match against). + const resultsIncludingSystem = await brain.find({ query: 'technology', limit: 10, includeSystem: true }) + expect(resultsIncludingSystem.some((r: any) => r.id === ROOT_ID)).toBe(false) + + await brain.close() + }) +}) From 2496e09aeb53521a971bfa4c2265fe6342720799 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 12:07:09 -0700 Subject: [PATCH 031/150] fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller --- src/brainy.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/brainy.ts b/src/brainy.ts index b498e632..bfac8c08 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1815,7 +1815,11 @@ export class Brainy implements BrainyInterface { if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { throw error } - throw new Error(`Failed to initialize Brainy: ${error}`) + // Wrap with the original as `cause` so the originating frame (a plugin's + // own file:line, e.g. a provider boot failure) survives to the caller's + // log — a plain string interpolation discards both stack and cause. + const message = error instanceof Error ? error.message : String(error) + throw new Error(`Failed to initialize Brainy: ${message}`, { cause: error }) } } From f8d8ce16b94b29041b64a12dfeac8ed6e6ce0dbd Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 12:38:52 -0700 Subject: [PATCH 032/150] fix(storage): enumeration re-keys on the identity record, not the vector leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The noun/verb pagination walks (getNounsWithPagination, getNounIdsWithPagination, getVerbsWithPagination) listed shard contents by filtering for vectors.json, while the canonical count ledger has always counted a row by its metadata.json presence alone. A row with metadata and no vector file was therefore counted by the ledger but never yielded by the walk — a permanent "counted but invisible" phantom for any downstream consumer that iterates the walk to account for the ledger's total. Nouns now enumerate by metadata.json and hydrate the vector leg optionally, yielding the sanctioned unvectored shape (vector: []) when it's absent. Verbs enumerate the same way, but a metadata-only verb row can only be fully reconstructed when sourceId/targetId happen to be recoverable from metadata (never true for a current production write — those fields live only in the vector leg); otherwise the row is counted but loudly skipped rather than fabricated, since a phantom edge with fake endpoints would be worse than the original defect. Separately, GenerationStore's recovery-fold replay (replayFact) now applies preserve-if-absent: a metadata-only after-image replayed over an already- vectored row carries the existing vector forward instead of deleting it via writeNounRaw/writeVerbRaw's exact-restore null-means-delete contract (which must stay exact for transaction-abort rollback). A genuine tombstone still removes both legs. --- src/db/generationStore.ts | 41 ++- src/storage/baseStorage.ts | 170 +++++++-- .../enumeration-population-law.test.ts | 333 ++++++++++++++++++ 3 files changed, 519 insertions(+), 25 deletions(-) create mode 100644 tests/integration/enumeration-population-law.test.ts diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index bfb68959..81bed338 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -654,10 +654,43 @@ export class GenerationStore { let replayed = 0 const replayFact = async (fact: CommitFact): Promise => { for (const op of fact.ops) { - const image = - op.record === null - ? { metadata: null, vector: null } - : { metadata: op.record.metadata, vector: op.record.vector } + let image: { metadata: unknown | null; vector: unknown | null } + if (op.record === null) { + // A genuine tombstone (both legs absent) — the fold removes + // both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw + // exact-restore contract. + image = { metadata: null, vector: null } + } else if ( + op.record.metadata !== null && + (op.record.vector === null || op.record.vector === undefined) + ) { + // PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's + // half): a metadata-only after-image must never DELETE an + // existing vector leg through the fold. `writeNounRaw`/ + // `writeVerbRaw` are exact-restore primitives — a `vector: + // null` there means "delete", which is exactly right for + // `rollBackUncommittedGeneration`'s before-image restore (a + // transaction abort legitimately un-writes a vector the failed + // transaction added). It is NOT right here: this fold replays + // AFTER-IMAGES, and re-applying an already-intact record must + // be byte-safe (this module's own invariant, see the log-authority + // comment above) — silently erasing a landed vector because one + // replayed fact's vector leg came back null is the exact defect + // that left metadata-counted, never-enumerated rows in a + // production store (confirmed root cause: the enumeration walk + // used to key on the vector leg, so a preserved-but-then-deleted + // vector made the row invisible while the ledger still counted + // it by metadata). A genuine "unvector" has its own sanctioned, + // ledger-correct path (`Brainy.unvectorNounForRootMigration`) — + // never this raw primitive, and never the fold. + const current = + op.kind === 'verb' + ? await this.storage.readVerbRaw(op.id) + : await this.storage.readNounRaw(op.id) + image = { metadata: op.record.metadata, vector: current.vector ?? null } + } else { + image = { metadata: op.record.metadata, vector: op.record.vector } + } if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) else await this.storage.writeNounRaw(op.id, image) this.noteCheckpointDirty(op.kind, op.id) diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index d6ccc5fa..ea88921c 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -203,6 +203,40 @@ function idFromVectorPath(path: string): string { return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix } +/** + * @description Extract the entity id embedded in a metadata path + * (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) — the IDENTITY-RECORD + * mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their + * population on this file (ADR-008 G1: the metadata record IS the population; + * the vector leg is optional), so walk ordering and cursor resume derive the + * id from THIS path, never the vector path — a row with metadata and no + * vector file must still be listed, ordered, and resumable. + * @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`). + * @returns The entity id (the path segment immediately before `/metadata.json`). + */ +function idFromMetadataPath(path: string): string { + const withoutSuffix = path.replace(/\/metadata\.json$/, '') + const lastSlash = withoutSuffix.lastIndexOf('/') + return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix +} + +/** + * @description The sanctioned UNVECTORED shape for a noun hydrated during + * enumeration when its identity record (metadata.json) exists but its vector + * leg (vectors.json) does not — a fold-born metadata-only after-image, or any + * row genuinely without a vector yet. Mirrors the shape + * `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned + * unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a + * walk-yielded unvectored row is byte-shape-identical to one produced by that + * migration. Callers already handle `vector: []` as first-class + * (validateAddParams exempts it; index gates key on `length > 0`). + * @param id - The noun id. + * @returns A structurally-valid, vector-empty `HNSWNoun`. + */ +function unvectoredNoun(id: string): HNSWNoun { + return { id, vector: [], connections: new Map>(), level: 0 } +} + /** * Get ID-first path for verb metadata * No type parameter needed - direct O(1) lookup by ID @@ -1453,6 +1487,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { * rollups are derived state with their own rebuild paths * (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`). * + * EXACT-RESTORE PRIMITIVE — `vector: null` DELETES the vector leg, on + * purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on + * this to legitimately un-write a vector a failed transaction added. This + * is deliberately NOT "preserve if absent" — a caller replaying an + * AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must + * apply preserve-if-absent itself BEFORE calling this, by reading the + * current vector and carrying it forward when the after-image's own + * vector leg is null/undefined but its metadata is not (see `replayFact` + * for the implementation and full rationale). A caller that genuinely + * wants to unvector a row uses the sanctioned, ledger-correct path + * (`Brainy.unvectorNounForRootMigration`) — never this primitive. + * * @param id - The entity id. * @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}. */ @@ -1489,7 +1535,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Restore a relationship's raw stored objects byte-for-byte (verb-side - * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats). + * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats, + * same EXACT-RESTORE contract — `vector: null` deletes, on purpose; the + * fold's preserve-if-absent logic lives at its call site, not here). * * @param id - The relationship id. * @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}. @@ -2183,9 +2231,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Stable within-shard order (by noun id) so offset windows and cursor resume // are deterministic; ids come from the path so skipped nouns are never read. + // + // IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record + // (not the vector) IS the population — a noun with metadata and no vector + // file (a fold-born after-image, see writeNounRaw's preserve-if-absent + // contract) must still enumerate. Keying on metadata.json here means the + // ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed) + // and this walk agree on population by construction. Ordering is + // unaffected for a healthy store: every vectored noun has both legs, so + // the id set and sort order are identical to the old vectors.json keying. const entries = nounFiles - .filter((p) => p.includes('/vectors.json')) - .map((p) => ({ path: p, id: idFromVectorPath(p) })) + .filter((p) => p.includes('/metadata.json')) + .map((p) => ({ path: p, id: idFromMetadataPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor @@ -2210,13 +2267,24 @@ export abstract class BaseStorage extends BaseStorageAdapter { ) { const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) const hydrated = await Promise.all( - batch.map(async ({ path: nounPath }) => { + batch.map(async ({ path: metadataPath, id }) => { try { - const noun = await this.readCanonicalObject(nounPath) - if (!noun) return null - const deserialized = this.deserializeNoun(noun) - const metadata = await this.getNounMetadata(deserialized.id) + const metadata = await this.readCanonicalObject(metadataPath) if (!metadata) return null + // The vector leg is OPTIONAL (population law): a metadata-only + // row hydrates with the sanctioned unvectored shape rather than + // being dropped from the walk. A fault reading the vector leg + // is treated the same as absence — best-effort, matching the + // canonical recount's tolerance for an unreadable vectors.json + // (rebuildTypeCounts) — a vector-leg problem never hides an + // otherwise-good identity record. + let deserialized: HNSWNoun + try { + const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id)) + deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id) + } catch { + deserialized = unvectoredNoun(id) + } return { deserialized, metadata } } catch (error) { // A TORN record must surface typed — a paginated read that @@ -2226,7 +2294,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { // walk's job is to HEAL PAST it — skip the victim, serve the rest. // Identity point-reads (get-by-id) still throw typed upstream. if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } - // Skip nouns that fail to load + // Skip nouns whose IDENTITY record fails to load (the metadata + // read above) — that is the one leg this walk cannot proceed + // without. return null } }) @@ -2347,9 +2417,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/nouns/${shardHex}` try { const nounFiles = await this.listCanonicalObjects(shardDir) + // IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching + // comment in getNounsWithPagination: metadata.json is the population; + // the vector leg is optional, so a metadata-only row must still be + // listed (and here, for the unfiltered case, needs ZERO reads either + // way — the id comes straight from the path). const entries = nounFiles - .filter((p) => p.includes('/vectors.json')) - .map((p) => idFromVectorPath(p)) + .filter((p) => p.includes('/metadata.json')) + .map((p) => idFromMetadataPath(p)) .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) const toWalk = cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries @@ -2560,23 +2635,79 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Stable within-shard order (by verb id) so offset windows and cursor resume // are deterministic and consistent across calls. Ids come from the path, so // verbs skipped by the cursor are never read. + // + // IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of + // this comment in getNounsWithPagination applies here too: metadata.json + // is the population; keying on it here means this walk and the ledger + // recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed) + // agree on population by construction. Unchanged for a healthy store — + // `relate()` always writes both legs of a verb in the same commit, so + // the id set and order match the old vectors.json keying exactly; this + // only additionally surfaces a fold-born metadata-only row (see + // writeVerbRaw's preserve-if-absent contract). const entries = verbFiles - .filter((p) => p.includes('/vectors.json')) - .map((p) => ({ path: p, id: idFromVectorPath(p) })) + .filter((p) => p.includes('/metadata.json')) + .map((p) => ({ path: p, id: idFromMetadataPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) - for (const { path: verbPath, id: verbId } of entries) { + for (const { path: metadataPath, id: verbId } of entries) { if (collected.length >= peekCount) break // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id // (later shards are processed in full). No read for skipped verbs. if (cursor && shard === cursor.shard && verbId <= cursor.id) continue try { - const rawVerb = await this.readCanonicalObject(verbPath) - if (!rawVerb) continue + // Identity leg first — required. A verb this walk cannot read + // metadata for cannot be hydrated at all (same as before). + const metadata = await this.readCanonicalObject(metadataPath) + if (!metadata) continue - // Deserialize connections Map from JSON storage format - const verb = this.deserializeVerb(rawVerb) + // The vector leg is the verb's STRUCTURAL core (verb/sourceId/ + // targetId live there — see coreTypes.ts HNSWVerb), unlike a + // noun's vector, which is pure embedding data. `relate()` always + // writes both legs atomically and verbs have no deferred-embed + // path, so a healthy store's verbs always have both. A vector-leg + // absence here can only be a fold-born after-image (see + // writeVerbRaw's preserve-if-absent contract) — and unlike a + // noun, this walk cannot safely FABRICATE sourceId/targetId to + // synthesize a structurally-valid verb (an empty-string endpoint + // would silently create a phantom edge — worse than omission). + // If the metadata record happens to carry its own sourceId/ + // targetId (never true for current production writes, but not + // disallowed — e.g. a future schema or a repair tool could + // populate them), reconstruct from those; otherwise this row is + // loudly skipped — counted by the ledger, but not returned as an + // item, until a repair can supply the missing endpoints. + const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId)) + let verb: HNSWVerb + if (rawVerb) { + verb = this.deserializeVerb(rawVerb) + } else { + const metaSourceId = (metadata as Record).sourceId + const metaTargetId = (metadata as Record).targetId + const metaVerbType = (metadata as Record).verb + if ( + typeof metaSourceId === 'string' && metaSourceId.length > 0 && + typeof metaTargetId === 'string' && metaTargetId.length > 0 && + typeof metaVerbType === 'string' && metaVerbType.length > 0 + ) { + verb = { + id: verbId, + vector: [], + connections: new Map>(), + verb: metaVerbType as VerbType, + sourceId: metaSourceId, + targetId: metaTargetId + } + } else { + prodLog.error( + `[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` + + `record but no vector leg and no recoverable sourceId/targetId — ` + + `skipping (counted by the ledger, not yielded; needs repair).` + ) + continue + } + } // Apply type filter if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) { @@ -2593,9 +2724,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { continue } - // Load metadata - const metadata = await this.getVerbMetadata(verb.id) - // Apply subtype filter (requires metadata — checked AFTER load) if (filterSubtypes) { const subtype = metadata?.subtype as string | undefined diff --git a/tests/integration/enumeration-population-law.test.ts b/tests/integration/enumeration-population-law.test.ts new file mode 100644 index 00000000..eaab7432 --- /dev/null +++ b/tests/integration/enumeration-population-law.test.ts @@ -0,0 +1,333 @@ +/** + * @module tests/integration/enumeration-population-law + * @description THE POPULATION LAW (ADR-008 G1): the unfiltered noun/verb walk + * and the canonical ALL scalar must agree on the population — a row's + * IDENTITY RECORD (metadata.json) is what defines membership; the vector leg + * is optional data, never a gate on visibility. Before this fix, the walk + * (getNounsWithPagination / getNounIdsWithPagination / getVerbsWithPagination) + * enumerated by keying on the VECTOR leg (`vectors.json`), so a row with + * metadata and no vector file was counted by the ledger (already + * metadata.json-keyed — see `rebuildTypeCounts`) but never yielded by the + * walk: a permanent "counted but invisible" phantom for any downstream + * consumer (a health-coverage row, an index-fill walk) that iterates the walk + * to account for the ledger's total. + * + * Two legs are pinned here: + * (a)/(b) LEG 1 — the walk re-keys on metadata.json. A fold-born + * metadata-only row (the exact shape `GenerationStore.replayFact` can + * leave behind, and the exact shape `writeNounRaw`/`writeVerbRaw` accept) + * must be YIELDED, hydrated with the sanctioned unvectored shape + * (`vector: []`) — not merely counted. + * + * For VERBS this closes only PARTIALLY: `sourceId`/`targetId` are + * HNSWVerb's structural core and live ONLY in the vector leg (never in + * metadata — see `RESERVED_RELATION_FIELDS` in reservedFields.ts, which + * does not include them). A metadata-only verb row therefore cannot be + * safely reconstructed without FABRICATING an edge's endpoints — which + * would silently create a phantom relationship, strictly worse than the + * original defect. The walk recovers the row when its metadata happens + * to carry `sourceId`/`targetId` (a defensive, forward-compatible + * fallback — never true for a CURRENT production write, but not + * disallowed either); otherwise it counts the row (ledger, unchanged) + * but loudly skips yielding it, logging the gap instead of hiding it. + * Closing this fully requires persisting `sourceId`/`targetId` in verb + * metadata — a schema change out of this task's scope; see the session + * report for the explicit call-out. + * + * (c)/(d) LEG 2 — the recovery fold's preserve-if-absent contract, exercised + * directly against `GenerationStore`/`FactLog` (below the `Brainy` API): + * a metadata-only after-image replayed over an already-vectored row must + * PRESERVE the existing vector leg (never delete it); a genuine tombstone + * (both legs absent) still removes both legs. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { randomUUID } from 'node:crypto' +import { Brainy } from '../../src/index.js' +import { GenerationStore } from '../../src/db/generationStore.js' +import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' +import { LOG_AUTHORITY_PATH } from '../../src/db/logAuthority.js' +import type { CommitFact } from '../../src/db/factLog.js' + +describe('enumeration population law — LEG 1 (identity-keyed walk)', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-population-law-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('(a) nouns.all equals the unfiltered walk-yield count with a fold-born metadata-only row present', async () => { + // Ordinary, fully-vectored background population. + await brain.add({ data: 'one', type: 'document' }) + await brain.add({ data: 'two', type: 'document' }) + await brain.flush() + + // THE EXACT PRE-FIX SHAPE: a metadata-only row against a FRESH id — no + // vector ever existed for it. Written through the raw primitive directly, + // exactly as `GenerationStore.replayFact` (the recovery fold) applies a + // replayed after-image whose vector leg came back null. + const freshId = randomUUID() + await brain.storage.writeNounRaw(freshId, { + metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 }, + vector: null + }) + + // writeNounRaw bypasses count bookkeeping on purpose (its own JSDoc) — the + // sanctioned recount brings the ledger scalar to ground truth. This walk + // was ALREADY metadata.json-keyed before this fix (rebuildTypeCounts), so + // the recount's answer does not depend on today's change. + await brain.repairIndex() + + const ledger = await brain.storage.getCanonicalCounts() + const walk = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } }) + + expect(walk.items.length).toBe(ledger.nouns.all) + expect(walk.totalCount).toBe(ledger.nouns.all) + + const yielded = walk.items.find((n: any) => n.id === freshId) + expect(yielded, 'the metadata-only row must be YIELDED, not merely counted').toBeDefined() + expect(yielded.vector).toEqual([]) + }) + + it('(a-ids) getNounIdsWithPagination (the zero-read unfiltered enumerator) also yields the metadata-only row', async () => { + await brain.add({ data: 'one', type: 'document' }) + await brain.flush() + + const freshId = randomUUID() + await brain.storage.writeNounRaw(freshId, { + metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 }, + vector: null + }) + await brain.repairIndex() + + const ledger = await brain.storage.getCanonicalCounts() + const page = await brain.storage.getNounIdsWithPagination({ limit: 1000, offset: 0 }) + expect(page.ids.length).toBe(ledger.nouns.all) + expect(page.ids).toContain(freshId) + }) + + it('(b) verbs.all counts a fold-born metadata-only row; the walk yields it when endpoints are recoverable from metadata, and loudly skips (never fabricates) when they are not', async () => { + const a = await brain.add({ data: 'a', type: 'document' }) + const b = await brain.add({ data: 'b', type: 'document' }) + await brain.relate({ from: a, to: b, type: 'relatedTo' }) + await brain.flush() + + // Case 1 — the REALISTIC production shape: metadata carries the verb + // type (a reserved field, kept for backward compat) but never + // sourceId/targetId — those are HNSWVerb's structural core and live + // ONLY in the vector leg. The walk cannot safely fabricate them (an + // empty-string endpoint would silently create a phantom edge), so this + // row is counted by the ledger but not yielded — a documented, + // loudly-logged gap, not a silent one. + const gapId = randomUUID() + await brain.storage.writeVerbRaw(gapId, { + metadata: { verb: 'relatedTo', createdAt: Date.now(), updatedAt: Date.now(), weight: 1 }, + vector: null + }) + + // Case 2 — endpoints ARE recoverable from metadata (never true for a + // current production write; modeled here as what a repair tool or a + // future schema could supply): the walk reconstructs and yields it. + const recoveredId = randomUUID() + await brain.storage.writeVerbRaw(recoveredId, { + metadata: { + verb: 'relatedTo', + sourceId: a, + targetId: b, + createdAt: Date.now(), + updatedAt: Date.now(), + weight: 1 + }, + vector: null + }) + + await brain.repairIndex() + const ledger = await brain.storage.getCanonicalCounts() + const walk = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } }) + + // The ledger counts every identity record — the real edge plus both + // synthetic metadata-only rows — unaffected by whether the walk can + // safely hydrate them. + expect(ledger.verbs.all).toBe(3) + + const recovered = walk.items.find((v: any) => v.id === recoveredId) + expect(recovered, 'endpoints recoverable from metadata must be yielded').toBeDefined() + expect(recovered.sourceId).toBe(a) + expect(recovered.targetId).toBe(b) + expect(recovered.vector).toEqual([]) + + // The documented gap: counted, not yielded — this is the one corner of + // the population law this task does NOT close (see the session report). + const gapped = walk.items.find((v: any) => v.id === gapId) + expect(gapped).toBeUndefined() + expect(walk.items.length).toBeLessThan(ledger.verbs.all) + }) +}) + +describe('enumeration population law — LEG 2 (fold preserve-if-absent, below the Brainy API)', () => { + /** A GenerationStore whose brain has already flipped to log authority — the + * precondition for `replayFact` (the recovery fold) to run at open(). */ + async function openLogAuthorityStore(): Promise<{ storage: MemoryStorage; store: GenerationStore }> { + const storage = new MemoryStorage() + await storage.init() + await storage.writeRawObject(LOG_AUTHORITY_PATH, { authority: 'log' }) + const store = new GenerationStore(storage) + await store.open() + return { storage, store } + } + + it('(c) nouns: a metadata-only after-image replayed over a vectored row PRESERVES the vector; it stays readable and the vectored ledger is untouched either way', async () => { + const { storage, store } = await openLogAuthorityStore() + const id = randomUUID() + const vectorRecord = { id, vector: [0.1, 0.2, 0.3], connections: {}, level: 0 } + + // Generation 1 — a real, honest commit: both legs land together. + await store.commitTransaction({ + touched: { nouns: [id], verbs: [] }, + execute: async () => { + await storage.writeNounRaw(id, { + metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 }, + vector: vectorRecord + }) + } + }) + const beforeVectoredCount = (await storage.getCanonicalCounts()).vectors.all + + // THE ANOMALOUS FACT, crafted directly (bypassing commitTransaction, + // whose honest read-after-write could never produce this on its own): + // metadata changed, vector leg null, while the row is STILL vectored on + // disk. This is exactly the shape the recovery fold must tolerate — + // modeling the confirmed production defect at the replay boundary. + const factLog = store.getFactLog()! + const anomalousFact: CommitFact = { + generation: 2, + timestamp: Date.now(), + ops: [ + { + kind: 'noun', + id, + record: { + metadata: { noun: 'document', createdAt: 1000, updatedAt: 2000, _rev: 2 }, + vector: null + } + } + ] + } + await factLog.append(anomalousFact) + await factLog.sync() + + // Reopen — a fresh GenerationStore over the SAME storage. Generation 2's + // fact sits above the (still generation-1) manifest, so it replays + // through the recovery fold — `replayFact`'s own call site. + const store2 = new GenerationStore(storage) + await store2.open() + + const after = await storage.readNounRaw(id) + expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull() + expect((after.vector as { vector: number[] }).vector).toEqual([0.1, 0.2, 0.3]) + expect((after.metadata as { updatedAt: number }).updatedAt).toBe(2000) // the new metadata DID apply + + // writeNounRaw bypasses ledger bookkeeping either way (by design — see + // its JSDoc), so this scalar is unaffected by the replay regardless of + // outcome; asserted for completeness against the task's exact wording. + const afterVectoredCount = (await storage.getCanonicalCounts()).vectors.all + expect(afterVectoredCount).toBe(beforeVectoredCount) + }) + + it('(c-verb) verbs: a metadata-only after-image replayed over a vectored edge PRESERVES the vector leg (sourceId/targetId/verb intact)', async () => { + const { storage, store } = await openLogAuthorityStore() + const id = randomUUID() + const sourceId = randomUUID() + const targetId = randomUUID() + const vectorRecord = { id, vector: [0.7, 0.8], connections: {}, verb: 'relatedTo', sourceId, targetId } + + await store.commitTransaction({ + touched: { nouns: [], verbs: [id] }, + execute: async () => { + await storage.writeVerbRaw(id, { + metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 1000, weight: 1 }, + vector: vectorRecord + }) + } + }) + + const factLog = store.getFactLog()! + const anomalousFact: CommitFact = { + generation: 2, + timestamp: Date.now(), + ops: [ + { + kind: 'verb', + id, + record: { + metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 2000, weight: 2 }, + vector: null + } + } + ] + } + await factLog.append(anomalousFact) + await factLog.sync() + + const store2 = new GenerationStore(storage) + await store2.open() + + const after = await storage.readVerbRaw(id) + expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull() + expect((after.vector as { sourceId: string }).sourceId).toBe(sourceId) + expect((after.vector as { targetId: string }).targetId).toBe(targetId) + expect((after.metadata as { weight: number }).weight).toBe(2) + }) + + it('(d) a genuine tombstone replay removes BOTH legs (never preserved)', async () => { + const { storage, store } = await openLogAuthorityStore() + const id = randomUUID() + const vectorRecord = { id, vector: [0.4, 0.5, 0.6], connections: {}, level: 0 } + + await store.commitTransaction({ + touched: { nouns: [id], verbs: [] }, + execute: async () => { + await storage.writeNounRaw(id, { + metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 }, + vector: vectorRecord + }) + } + }) + expect((await storage.readNounRaw(id)).vector).not.toBeNull() // sanity: it landed + + const factLog = store.getFactLog()! + await factLog.append({ + generation: 2, + timestamp: Date.now(), + ops: [{ kind: 'noun', id, record: null }] // a genuine tombstone — both legs absent + }) + await factLog.sync() + + const store2 = new GenerationStore(storage) + await store2.open() + + const after = await storage.readNounRaw(id) + expect(after.metadata, 'a genuine delete removes the metadata leg').toBeNull() + expect(after.vector, 'a genuine delete removes the vector leg too — preserve-if-absent never applies to a tombstone').toBeNull() + }) +}) From fd6b4ce4ffc4194df1311dc8cb7900b2e9f67dd0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 13:00:44 -0700 Subject: [PATCH 033/150] fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load --- src/storage/adapters/baseStorageAdapter.ts | 12 ++ src/storage/adapters/fileSystemStorage.ts | 81 +++++++- src/storage/baseStorage.ts | 4 + .../ledger-derivation-identity.test.ts | 195 ++++++++++++++++++ 4 files changed, 285 insertions(+), 7 deletions(-) create mode 100644 tests/integration/ledger-derivation-identity.test.ts diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 88018a66..cabe2e30 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1066,6 +1066,18 @@ export abstract class BaseStorageAdapter implements StorageAdapter { protected allCountsSuspect = false /** One narration per session for the suspect transition (never per delete). */ private allCountsSuspectNarrated = false + /** + * Which rule produced the ALL scalars currently in memory. `'identity-record'` + * means one counted entity per metadata content leg — the honest rule: a + * bare id-directory (a ghost or scar left by a partial-delete defect, no + * content leg) counts zero. Set by the one-time derivation and by the + * sanctioned recount, alongside `allCountsSuspect = false`; left `undefined` + * when a loaded counts.json carries the ALL scalars but no stamp — the + * legacy container-rule derivation, which forces `allCountsSuspect = true` + * at load instead. A filesystem concern: `MemoryStorage` has no counts.json + * and never sets this. + */ + protected allCountsDerivedBy?: 'identity-record' protected entityCounts: Map = new Map() // type -> count protected verbCounts: Map = new Map() // verb type -> count protected countCache: Map = new Map() diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index fd9dbb4c..817e5e36 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -18,6 +18,7 @@ import { } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' +import { prodLog } from '../../utils/logger.js' import { TornRecordError, isUnparseablePayloadError, @@ -602,6 +603,20 @@ export class FileSystemStorage extends BaseStorage { * automatically. Returns the pruned container ids so the caller can recompute * counts. */ + /** + * @description Whether an id directory's file legs include the metadata + * CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single + * test that decides whether an `entities////` container is + * a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete + * defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune + * and {@link scanCanonicalEntities} so the two agree by construction — one + * counted entity per identity record, never per bare container. + * @param legs - File names in one `entities////` directory. + */ + private hasMetadataContentLeg(legs: string[]): boolean { + return legs.some((f) => f.startsWith('metadata.json')) + } + public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { await this.ensureInitialized() const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } @@ -641,7 +656,7 @@ export class FileSystemStorage extends BaseStorage { } // A live entity has its metadata content leg. No content leg → a // vector-only ghost or an empty scar → prune the whole container. - if (legs.some((f) => f.startsWith('metadata.json'))) continue + if (this.hasMetadataContentLeg(legs)) continue await fs.promises.rm(idAbs, { recursive: true, force: true }) pruned[kind].push(entry.name) console.warn( @@ -2590,13 +2605,36 @@ export class FileSystemStorage extends BaseStorage { ) { this.totalNounCountAll = counts.totalNounCountAll this.totalVerbCountAll = counts.totalVerbCountAll - this.allCountsSuspect = counts.allCountsSuspect === true + if (counts.allCountsDerivedBy === 'identity-record') { + // Derived (or recounted) under the honest rule — one counted + // entity per metadata content leg. Trust the persisted suspect + // flag as-is; an unprovable delete since may still have set it. + this.allCountsDerivedBy = 'identity-record' + this.allCountsSuspect = counts.allCountsSuspect === true + } else { + // The ALL scalars exist but predate the identity-record stamp — + // they were derived under the legacy rule that counted one + // entity per id DIRECTORY, so orphaned ghost/scar containers (a + // pre-8.3.1 partial-delete defect — see pruneOrphanedEntities()) + // were counted as entities too. O(1) field read, NEVER a walk + // here: force suspect and name it loudly. A sanctioned recount + // (repairIndex) restores exact denominators and clears this. + this.allCountsDerivedBy = undefined + this.allCountsSuspect = true + needsPersist = true + prodLog.warn( + '[FileSystemStorage] canonical count ledger was derived under the legacy ' + + 'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' + + 'exact denominators' + ) + } } else { const nouns = await this.scanCanonicalEntities('nouns') const verbs = await this.scanCanonicalEntities('verbs') this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false + this.allCountsDerivedBy = 'identity-record' console.warn( `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` + `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + @@ -2667,6 +2705,7 @@ export class FileSystemStorage extends BaseStorage { this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false + this.allCountsDerivedBy = 'identity-record' // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT // (a deferred-embed noun's file exists but holds an empty vector until // its embed lands), so this is a full O(nouns) content scan — see @@ -2704,10 +2743,19 @@ export class FileSystemStorage extends BaseStorage { /** * Walk the canonical `entities//<2-hex-shard>//` tree, counting - * one entity per id directory (the layout `getNounVectorPath`/`getNouns` - * use). Returns up to 100 sampled entity directories (absolute paths) — - * nouns feed the type-distribution estimate above. An absent tree (fresh - * store) counts zero. + * one entity per id directory that holds the metadata CONTENT leg + * (`metadata.json` or its `.json.gz` variant — see + * {@link hasMetadataContentLeg}). A bare container — a ghost (a stale + * `vectors.json` left with no metadata leg) or a scar (an empty directory), + * both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the + * identity record IS the population (ADR-008 G1), never the directory. + * This is the ONE-TIME legacy derivation walk (see callers); a prior + * version of this scan counted every id directory regardless of content, + * over-counting any store carrying orphaned containers — see + * `allCountsDerivedBy` for how a counts.json derived under that old rule is + * marked suspect on load. Returns up to 100 sampled *counted* entity + * directories (absolute paths) — nouns feed the type-distribution estimate + * above. An absent tree (fresh store) counts zero. */ private async scanCanonicalEntities( kind: 'nouns' | 'verbs' @@ -2724,9 +2772,21 @@ export class FileSystemStorage extends BaseStorage { const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) for (const entry of ids) { if (!entry.isDirectory()) continue + const idAbs = path.join(shardPath, entry.name) + let legs: string[] + try { + legs = await fs.promises.readdir(idAbs) + } catch (error: any) { + if (error?.code === 'ENOENT') continue + throw error + } + // No metadata content leg → a ghost or scar container → not an + // entity. Same test pruneOrphanedEntities() uses, so the two agree + // by construction. + if (!this.hasMetadataContentLeg(legs)) continue count++ if (sampleDirs.length < SAMPLE_MAX) { - sampleDirs.push(path.join(shardPath, entry.name)) + sampleDirs.push(idAbs) } } } @@ -2834,6 +2894,13 @@ export class FileSystemStorage extends BaseStorage { // scanVectoredNounCount()'s JSDoc). totalVectoredNounCount: this.totalVectoredNounCount, allCountsSuspect: this.allCountsSuspect, + // Derivation-rule stamp for the ALL scalars above — 'identity-record' + // when they were counted one-per-metadata-content-leg (the honest + // rule); omitted (JSON.stringify drops `undefined`) when the current + // in-memory scalars came from a legacy container-rule counts.json + // that hasn't been through a sanctioned recount yet, so a future load + // keeps naming them suspect rather than trusting an unproven value. + allCountsDerivedBy: this.allCountsDerivedBy, lastUpdated: new Date().toISOString() } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index ea88921c..5510f93b 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -4820,6 +4820,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.totalVerbCountAll = allVerbs this.totalVectoredNounCount = allVectoredNouns this.allCountsSuspect = false + // This walk counts one entity per metadata.json record (never per bare + // container) — the identity-record rule. Stamp it so a future load + // trusts these scalars instead of naming them suspect at open. + this.allCountsDerivedBy = 'identity-record' this.countCache.clear() await this.persistCounts() diff --git a/tests/integration/ledger-derivation-identity.test.ts b/tests/integration/ledger-derivation-identity.test.ts new file mode 100644 index 00000000..cb19af4a --- /dev/null +++ b/tests/integration/ledger-derivation-identity.test.ts @@ -0,0 +1,195 @@ +/** + * @module tests/integration/ledger-derivation-identity + * @description The ALL-visibility ledger scalars are an IDENTITY-RECORD + * count, never a container count. A pre-8.3.1 partial-delete defect can + * leave a "ghost" container (a stale `vectors.json` with no metadata content + * leg) or a "scar" container (an empty `entities////` + * directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need + * the metadata content leg — yet the legacy derivation counted one entity + * per id DIRECTORY, so orphaned containers inflated the ALL scalars forever + * (they were never clamped and never re-derived). Laws under test: + * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per + * metadata content leg (`metadata.json` or `.json.gz`), the same test + * `pruneOrphanedEntities()` uses, so the two agree by construction. + * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that + * carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'` + * stamp predates this fix; loading it marks `suspect = true` from a + * single field read alone, never a directory walk, and warns exactly + * once naming the cause. + * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned + * containers, recounts from the canonical metadata.json walk, and + * re-stamps — suspect clears and the ALL scalar is exact again. + * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store + * with no counts.json stamps as it writes, so a brand-new store never + * carries the legacy signature. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, FileSystemStorage } from '../../src/index.js' +import { prodLog } from '../../src/utils/logger.js' + +const countsPath = (root: string) => path.join(root, '_system', 'counts.json') + +/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */ +function plantGhost(root: string, shard: string, id: string): void { + const idDir = path.join(root, 'entities', 'nouns', shard, id) + fs.mkdirSync(idDir, { recursive: true }) + fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] })) +} + +/** Plant a scar container: an empty id directory, no legs at all. */ +function plantScar(root: string, shard: string, id: string): void { + fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true }) +} + +describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => { + let dir: string + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(() => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-')) + }) + afterEach(() => { + vi.restoreAllMocks() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => { + let brain = await open() + const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone + for (let i = 0; i < 3; i++) { + await brain.add({ data: `real ${i}`, type: 'document' }) + } + await brain.flush() + const realTotal = baseline + 3 + await brain.close() + + // 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) — + // neither is a live entity. + for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) + for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) + + // Remove counts.json so open() re-derives from scratch (the one-time + // legacy/lost-file derivation path). + fs.rmSync(countsPath(dir), { force: true }) + + brain = await open() + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing + expect(ledger.suspect).toBe(false) + + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.totalNounCountAll).toBe(realTotal) + expect(raw.allCountsDerivedBy).toBe('identity-record') + + await brain.close() + }) + + it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => { + let brain = await open() + await brain.add({ data: 'one', type: 'document' }) + await brain.add({ data: 'two', type: 'document' }) + await brain.flush() + await brain.close() + + // Confirm a normal close under the fix DOES stamp — then strip the stamp + // to simulate a counts.json produced before this fix existed. + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.allCountsDerivedBy).toBe('identity-record') + expect(typeof raw.totalNounCountAll).toBe('number') + expect(typeof raw.totalVerbCountAll).toBe('number') + expect(typeof raw.totalVectoredNounCount).toBe('number') + delete raw.allCountsDerivedBy + fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) + + const warnSpy = vi.spyOn(prodLog, 'warn') + // The two derivation walks live on FileSystemStorage's prototype — + // spying here (rather than on fs.promises.readdir globally) isolates + // THIS code path's behavior from unrelated walks elsewhere in the open + // sequence (a separate, pre-existing engine's own O(store) cost — not + // this fix's concern, and not something this pin should be sensitive + // to). Neither derivation method may run: the stamp check is a field + // read on the already-parsed counts.json, nothing more. + const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities') + const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount') + + brain = await open() + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(true) + + const stampWarnings = warnSpy.mock.calls.filter( + ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule') + ) + expect(stampWarnings.length).toBe(1) // exactly one, loud + + expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk + expect(scanVectoredSpy).not.toHaveBeenCalled() + + await brain.close() + }) + + it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => { + let brain = await open() + const baseline = (await brain.storage.getCanonicalCounts()).nouns.all + for (let i = 0; i < 3; i++) { + await brain.add({ data: `real ${i}`, type: 'document' }) + } + await brain.flush() + const realTotal = baseline + 3 + await brain.close() + + for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) + for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) + + // Force the legacy (unstamped, container-rule-inflated) shape directly — + // the shape a pre-existing production store actually carries. + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars + delete raw.allCountsDerivedBy + fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) + + brain = await open() + expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load + + await brain.repairIndex() + + let ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again + + const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(persisted.allCountsDerivedBy).toBe('identity-record') + expect(persisted.allCountsSuspect).toBe(false) + expect(persisted.totalNounCountAll).toBe(realTotal) + + await brain.close() + brain = await open() + ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.nouns.all).toBe(realTotal) + await brain.close() + }) + + it('(d) a fresh store derives with the stamp and is never suspect', async () => { + const brain = await open() + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.allCountsDerivedBy).toBe('identity-record') + await brain.close() + }) +}) From 8fc553b126fc0ef0dc0d2be29e71db4143b93f1a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 13:29:11 -0700 Subject: [PATCH 034/150] fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index A canonical row persisted with vector: [] (a system row, a deferred embed not yet landed, or any other legitimately-unvectored record) is a normal, enumerable row -- but rebuild()'s storage walk had no guard against it. storage.getVectorIndexData() derives its answer from the row's own record, so it returns non-null for any existing noun whether or not that noun was ever actually indexed -- rebuild() admitted such rows into the live graph with a length-0 vector. A vector-less node could become the entry point (or occupy any graph position); the next real insert then ran a distance calculation against it and blew up with a dimension mismatch. Fix at two layers in src/hnsw/hnswIndex.ts: - rebuild() now skips any row whose vector.length === 0 before it ever becomes a graph node (one summary count line, never per-row spam), and restores the pinned dimension from the first real vector it loads -- previously the pin stayed null across a restart, since addItem/updateItem are the only sites that set it and rebuild() never goes through either. - addItem/updateItem now refuse a length-0 vector with a typed EmptyVectorIndexError instead of ever pinning dimension to 0 or storing a vector-less node, so no future fill/rebuild/load path can poison the index silently. getVectorSafe's lazy-load "not found" check also missed that an empty array is truthy -- tightened to catch it. IndexOperations.ts's ReplaceInVectorIndexOperation rollback paths now skip re-adding an oldVector of length 0 (never a legal index member) instead of attempting an illegal empty re-insert on rollback. biography.test.ts's final ledger-exactness assertion assumed every noun the lane creates is vectored, including the VFS root counted in vfsBaselineNouns -- but the root is deliberately persisted unvectored. Corrected the expected formula to exclude it. Adds tests/integration/index-skips-unvectored.test.ts pinning: rebuild() indexes only vectored rows with the dimension pinned correctly; clear() then real adds never trip a dimension mismatch; addItem/updateItem refuse a length-0 vector; and a crash/repair cycle stays dimension-consistent. --- src/hnsw/hnswIndex.ts | 92 ++++++- src/transaction/operations/IndexOperations.ts | 22 +- .../index-skips-unvectored.test.ts | 253 ++++++++++++++++++ tests/lifecycle/biography.test.ts | 12 +- 4 files changed, 371 insertions(+), 8 deletions(-) create mode 100644 tests/integration/index-skips-unvectored.test.ts diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 77e4f84d..3caf23ba 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -64,6 +64,34 @@ export class HnswFlushError extends Error { } } +/** + * @description Thrown by {@link JsHnswVectorIndex.addItem} / {@link + * JsHnswVectorIndex.updateItem} when handed a length-0 vector. A length-0 + * vector is the sanctioned "unvectored" shape for a canonical noun record + * (class-J: a VFS-system row, a deferred embed not yet landed, or any other + * legitimately-vector-less row) — but it is NEVER a legal INDEX insert. The + * index itself has no concept of "unvectored"; deciding that a row is + * unvectored and therefore skippable is the FILL/REBUILD/LOAD consumer's job + * (see {@link JsHnswVectorIndex.rebuild}), done BEFORE ever calling addItem. + * A length-0 vector reaching this point is a caller bug: silently accepting + * it would pin `this.dimension = 0` on an empty index (poisoning every real + * insert thereafter with a dimension mismatch) or store a vector-less node + * that a distance calculation can never safely compare against. Loud errors, + * never quiet losses — this throws instead of either. + */ +export class EmptyVectorIndexError extends Error { + constructor(public readonly id: string, operation: 'addItem' | 'updateItem') { + super( + `${operation}(${id}): refusing to index a length-0 vector — a length-0 vector is the ` + + `sanctioned "unvectored" shape for a canonical row, but it is never a legal index ` + + `insert. Callers that fill/rebuild/load the index must skip vector.length === 0 rows ` + + `themselves (unvectored = nothing to index, not an error at that layer); reaching ` + + `here with one is a caller bug.` + ) + this.name = 'EmptyVectorIndexError' + } +} + /** * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native @@ -580,6 +608,15 @@ export class JsHnswVectorIndex implements VectorIndexProvider { throw new Error('Vector is undefined or null') } + // THE INDEX REFUSES A LENGTH-0 VECTOR (see EmptyVectorIndexError's JSDoc): + // an empty vector is the sanctioned "unvectored" shape at the canonical + // layer, never a legal index member. Refusing here — loudly, before the + // dimension pin below — means no future fill/rebuild/load path can ever + // poison `this.dimension` to 0 or park a vector-less node in the graph. + if (vector.length === 0) { + throw new EmptyVectorIndexError(id, 'addItem') + } + // Set dimension on first insert if (this.dimension === null) { this.dimension = vector.length @@ -954,6 +991,13 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return } + // Same refusal as addItem (see EmptyVectorIndexError's JSDoc) — an + // in-place relink must never rewrite an already-indexed node down to the + // unvectored shape or poison the pinned dimension. + if (vector.length === 0) { + throw new EmptyVectorIndexError(id, 'updateItem') + } + if (this.dimension === null) { this.dimension = vector.length } else if (vector.length !== this.dimension) { @@ -1555,7 +1599,15 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } const loaded = await this.storage.getNounVector(noun.id) - if (!loaded) { + // `loaded` is a length-0 array (not null/undefined) for a canonical row + // that is legitimately unvectored — `![]` is FALSE (an empty array is + // truthy), so the bare `!loaded` check below would silently accept it + // as "found" and hand a dimension-0 vector to a distance calculation. + // A node only reaches this lazy-load path because it is a MEMBER of + // the live index (rebuild() now refuses to admit unvectored rows — see + // its JSDoc), so an empty vector here is never legitimate: treat it + // exactly like "not found", loudly. + if (!loaded || loaded.length === 0) { throw new Error(`Vector not found for noun ${noun.id}`) } @@ -1765,9 +1817,42 @@ export class JsHnswVectorIndex implements VectorIndexProvider { totalCount = result.totalCount || result.items.length + // UNVECTORED ROWS ARE NOT AN INDEX MEMBER (the class-J law): a canonical + // noun whose vector leg is `[]` (a VFS-root-style system row, a + // deferred embed not yet landed, or a best-effort fallback for an + // unreadable vector leg) is a normal, enumerable, countable row — it + // is simply not indexed. `storage.getVectorIndexData()` derives its + // {level, connections} answer straight from the noun's OWN record, so + // it returns non-null for every existing noun regardless of whether + // that noun ever actually reached `addItem()` — it cannot be used to + // decide indexability. `nounData.vector.length === 0` is the one + // truthful signal (mirrors the `noun.vector.length > 0` guards in + // {@link getVectorSafe} / {@link getVectorSync}): skip here, counted + // once in a summary line, never per-row spam. + let skippedUnvectored = 0 + // Process all nouns at once for (const nounData of result.items) { try { + if (!nounData.vector || nounData.vector.length === 0) { + skippedUnvectored++ + continue + } + + // Restore the pinned dimension from the first real vector this + // rebuild loads. `addItem`/`updateItem` only pin `this.dimension` + // on a LIVE insert — a fresh rebuild from storage never goes + // through either, so without this the pin stays `null` across a + // restart. A `null` pin means the very next insert (correct OR + // wrong length) silently BECOMES the new pin instead of being + // checked against the store's real dimension — the wrong-length + // case then fails much later and less clearly, inside a distance + // calculation against an already-loaded node, instead of here, + // immediately, with a named expected-vs-got mismatch. + if (this.dimension === null) { + this.dimension = nounData.vector.length + } + // Load HNSW graph data for this entity const hnswData = await this.storage.getVectorIndexData(nounData.id) @@ -1815,7 +1900,10 @@ export class JsHnswVectorIndex implements VectorIndexProvider { options.onProgress(loadedCount, totalCount) } - prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`) + prodLog.info( + `HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})` + + (skippedUnvectored > 0 ? ` — ${skippedUnvectored.toLocaleString()} unvectored row(s) skipped` : '') + ) } // Step 5: CRITICAL - Recover entry point if missing) diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index cfee5074..1bbbca88 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -324,8 +324,17 @@ export class ReplaceInVectorIndexOperation implements Operation { return async () => { // Restore the declared before-state in place (see class JSDoc for - // the item-did-not-exist posture). - await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) + // the item-did-not-exist posture). A length-0 oldVector means the row + // was never actually indexed before this op ran (a length-0 vector is + // never a legal index member — see EmptyVectorIndexError) — there is + // no in-place "restore to empty" for the provider to perform, so + // rollback removes the row instead, leaving the same "not indexed" + // state the row was in before execute(). + if (this.oldVector.length > 0) { + await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) + } else { + await this.index.removeItem(this.id, generation) + } } } @@ -336,9 +345,14 @@ export class ReplaceInVectorIndexOperation implements Operation { return async () => { // updateItem-style restore via the same adjacent pair, back to the - // declared before-state. + // declared before-state. Same length-0 carve-out as the updateItem + // path above: an empty oldVector was never a legal index member, so + // rollback just leaves the row removed rather than attempting an + // illegal empty re-add. await this.index.removeItem(this.id, generation) - await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + if (this.oldVector.length > 0) { + await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + } } } } diff --git a/tests/integration/index-skips-unvectored.test.ts b/tests/integration/index-skips-unvectored.test.ts new file mode 100644 index 00000000..c65aaa01 --- /dev/null +++ b/tests/integration/index-skips-unvectored.test.ts @@ -0,0 +1,253 @@ +/** + * @module tests/integration/index-skips-unvectored + * @description THE UNVECTORED-ROW CURE — two integration tests + * (`tests/lifecycle/biography.test.ts`'s Ch4/5/6 chapter and + * `tests/integration/clear-persistence.test.ts`'s multi-cycle test) started + * failing after a canonical-storage change made a vector-less row (the + * class-J shape: `vector: []`, e.g. the VFS root, a deferred embed not yet + * landed, or any other legitimately-unvectored canonical record) VISIBLE to + * the enumeration walk `getNounsWithPagination()` for the first time — before + * that change such rows were simply invisible to the walk. `hnswIndex.ts`'s + * `rebuild()` never guarded against that shape: it inserted every row the + * walk yielded into the live in-memory index, including ones with a length-0 + * vector, because `storage.getVectorIndexData()` derives its {level, + * connections} answer straight from the noun's OWN record — it returns + * non-null for ANY existing noun, whether or not that noun was ever actually + * indexed via `addItem()`. A vector-less node admitted into the graph could + * become the entry point (or occupy any graph position), and the very next + * real-vectored `addItem()` then ran a distance calculation against it — + * `cosineDistance` throws "Vectors must have the same dimensions" the moment + * one operand is a length-0 array. + * + * THE FIX, at two layers (`src/hnsw/hnswIndex.ts`): + * (1) FILL/REBUILD/LOAD consumers treat `vector.length === 0` as "unvectored — + * nothing to index" and skip the row (normal, not an error; one summary + * count line, never per-row spam) — `rebuild()`'s loop now checks this + * BEFORE ever creating a graph node, so an unvectored row can never + * become an index member, entry point, or dimension-setter. + * (2) THE INDEX ITSELF refuses a length-0 vector in `addItem()` / + * `updateItem()` with a typed `EmptyVectorIndexError`, loudly, instead of + * ever pinning `dimension = 0` or storing a vector-less node — so no + * future fill/rebuild/load path can silently poison the index even if it + * forgets law (1). + * + * Four legs pinned here: + * (a) `rebuild()` over a store mixing real-vectored rows and `vector: []` + * rows indexes ONLY the vectored ones — size === vectored count, + * dimension pinned to the real (non-zero) length. + * (b) `clear()` then real adds afterward never trip a dimension mismatch — + * the exact `clear-persistence.test.ts` regression shape, reproduced + * directly against the index/storage seam this module owns. + * (c) `index.addItem({ id, vector: [] })` throws `EmptyVectorIndexError` + * (and `updateItem` does too, for an existing node). + * (d) crash -> repair: the crashed generation's entities survive, the ledger + * recounts honestly, and a fresh real-vectored add afterward never trips + * a dimension mismatch against a leftover vector-less phantom. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { EmptyVectorIndexError } from '../../src/hnsw/hnswIndex.js' +import { abandonAsCrashed, openBrain as openKillMatrixBrain, uid, vec } from '../helpers/durabilityKillMatrix.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) { + try { + fs.rmSync(d, { recursive: true, force: true }) + } catch { + /* best-effort cleanup */ + } + } +}) + +/** A filesystem-backed brain with explicit vectors (no embedder needed) and + * manual persistence — mirrors `durabilityKillMatrix.ts`'s `openBrain` so + * every write in this module is explicit and provably durable. */ +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + persistence: { policy: 'manual' } + }) +} + +describe('HNSW index skips unvectored rows', () => { + it('(a) rebuild() indexes only vectored rows: size === vectored count, dimension pinned to the real length', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() + + // 5 real-vectored rows. + const vectoredIds: string[] = [] + for (let i = 0; i < 5; i++) { + const id = uid(`vectored-${i}`) + await brain.add({ id, data: `real entity ${i}`, type: NounType.Document, vector: vec(i) }) + vectoredIds.push(id) + } + // 3 explicit unvectored rows — the class-J "vector: []" shape, a normal, + // enumerable, countable canonical row that must never reach the index. + const unvectoredIds: string[] = [] + for (let i = 0; i < 3; i++) { + const id = uid(`unvectored-${i}`) + await brain.add({ id, data: `unvectored entity ${i}`, type: NounType.Document, vector: [] }) + unvectoredIds.push(id) + } + await brain.flush() + + // The canonical ledger already agrees before any rebuild: nouns.all + // counts every row (8 + the VFS root); vectors.all counts only the real + // ones (5) — the VFS root and the 3 explicit unvectored rows are excluded. + const ledgerBeforeReopen = await brain.storage.getCanonicalCounts() + expect(ledgerBeforeReopen.vectors.all).toBe(5) + expect(ledgerBeforeReopen.nouns.all).toBe(9) // 5 vectored + 3 unvectored + 1 VFS root + + await brain.close() + + // Reopen: open()'s index build IS hnswIndex.rebuild() run fresh from + // storage — this is the exact path that used to admit unvectored rows. + brain = openBrain(dir) + await brain.init() + + const status = await brain.getIndexStatus() + expect(status.hnswIndex.size, 'the rebuilt index must contain ONLY the 5 real-vectored rows').toBe(5) + + // Dimension is pinned to the REAL embedded length (384 via `vec()`), not + // 0 — adding a wrong-length vector must be refused naming that real + // dimension, proving no vector-less row ever set it. + const realDimension = vec(0).length + let mismatchMessage: string | undefined + try { + await brain.index.addItem({ id: uid('dimension-probe'), vector: vec(0).slice(0, realDimension - 1) }) + expect.fail('expected a dimension mismatch error') + } catch (err) { + mismatchMessage = (err as Error).message + } + expect(mismatchMessage).toContain(`expected ${realDimension}`) + + // Every unvectored row is still a normal, enumerable, readable canonical + // record — class-J semantics survive the rebuild fix untouched. + for (const id of unvectoredIds) { + const entity = await brain.get(id, { includeVectors: true }) + expect(entity, `unvectored entity ${id} must remain readable`).not.toBeNull() + expect(entity.vector).toEqual([]) + } + // A correct-dimension add succeeds cleanly against the pinned dimension. + const freshId = uid('post-reopen-fresh') + await expect(brain.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(50) })).resolves.toBe( + freshId + ) + + await brain.close() + }) + + it('(b) clear() then real adds afterward never trip a dimension mismatch (the clear-persistence regression shape)', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() // the VFS root (vector: []) is the store's only row + + await brain.clear() + await brain.close() + + // Reopen over a store whose only surviving row is the recreated, + // unvectored VFS root — this is exactly the shape that used to poison + // the entry point / dimension in `clear-persistence.test.ts`. + brain = openBrain(dir) + await brain.init() + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(0) + + const id1 = uid('after-clear-1') + await expect(brain.add({ id: id1, data: 'after clear 1', type: NounType.Document, vector: vec(1) })).resolves.toBe( + id1 + ) + const id2 = uid('after-clear-2') + await expect(brain.add({ id: id2, data: 'after clear 2', type: NounType.Document, vector: vec(2) })).resolves.toBe( + id2 + ) + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(2) + + await brain.close() + }) + + it('(c) index.addItem/updateItem refuse a length-0 vector with EmptyVectorIndexError', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + await expect(brain.index.addItem({ id: uid('empty-add'), vector: [] })).rejects.toThrow(EmptyVectorIndexError) + + // updateItem on an EXISTING (real-vectored) node must refuse the same way. + const existingId = uid('existing-for-update') + await brain.add({ id: existingId, data: 'existing', type: NounType.Document, vector: vec(9) }) + await expect(brain.index.updateItem({ id: existingId, vector: [] })).rejects.toThrow(EmptyVectorIndexError) + + // The index was never disturbed by either refused call. + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(1) + + await brain.close() + }) + + it('(d) crash -> repair: the crashed generation survives, the ledger recounts honestly, and a fresh add afterward never trips a dimension mismatch', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-crash-')) + try { + let brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) + + // Baseline: real-vectored entities, durably flushed. + const baselineIds: string[] = [] + for (let i = 0; i < 5; i++) { + const id = uid(`baseline-${i}`) + await brain.add({ id, data: `baseline entity ${i}`, type: NounType.Document, vector: vec(i) }) + baselineIds.push(id) + } + await brain.flush() + + // Crash window: at-ack writes that are never flushed before the crash. + const crashedIds: string[] = [] + for (let i = 0; i < 4; i++) { + const id = uid(`crashed-${i}`) + await brain.add({ id, data: `crash-window entity ${i}`, type: NounType.Document, vector: vec(100 + i) }) + crashedIds.push(id) + } + await abandonAsCrashed(brain) + + // Reopen — logAuthority: 'adopt' replays the at-ack log for the crash window. + brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) + for (const id of [...baselineIds, ...crashedIds]) { + expect(await brain.get(id), `entity ${id} must survive the crash`).not.toBeNull() + } + + // Repair — must not disturb any entity, and must recount the ledger honestly. + const report = await brain.repairIndex() + expect(report.families.length).toBeGreaterThan(0) + for (const id of [...baselineIds, ...crashedIds]) { + expect(await brain.get(id), `entity ${id} must survive repair`).not.toBeNull() + } + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.vectors.all).toBe(baselineIds.length + crashedIds.length) + + // Second life: a fresh real-vectored add must never trip a dimension + // mismatch against a vector-less phantom left in the index — the exact + // mechanism `clear-persistence.test.ts` and the biography lane hit. + const secondLifeId = uid('second-life') + await expect( + brain.add({ id: secondLifeId, data: 'second life entity', type: NounType.Document, vector: vec(200) }) + ).resolves.toBe(secondLifeId) + expect(await brain.get(secondLifeId)).not.toBeNull() + + await brain.close() + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts index 8b274fce..274f0ef0 100644 --- a/tests/lifecycle/biography.test.ts +++ b/tests/lifecycle/biography.test.ts @@ -382,9 +382,17 @@ describe.sequential('lifecycle — the working store', () => { }, // Every noun this biography ever adds carries an explicit/computed // vector (the harness never defers an embed), so the vectored-noun - // scalar tracks nouns.all exactly. + // scalar tracks nouns.all exactly EXCEPT for the VFS root counted + // in `vfsBaselineNouns`: the root is deliberately persisted with + // `vector: []` (the sanctioned "unvectored" shape — see + // VirtualFileSystem.doInitializeRoot()'s zero-norm-avoidance + // comment) so it never pays the WASM engine's cold-compile cost and + // never crosses an engine boundary as a false attractor. It is the + // ONE hidden-tier record `vfsBaselineNouns` represents (see + // biographyHarness's module header), so it is excluded here even + // though it counts toward `nouns.all`. vectors: { - all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns + all: aliveEntities.length + model.vfsFileNouns }, suspect: false }) From 0de7665930ac23fb48cff0c0034f4fb475527727 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 13:53:09 -0700 Subject: [PATCH 035/150] fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine-pair seam law: a zero-norm vector never crosses an engine boundary. The index belt already refused to insert one, but the canonical write and the vectored-noun ledger still counted it, so a near-empty store whose only vectored row was zero-norm read "1 canonical vectored vs 0 indexed" and threw a not-ready error at open, and a store's own legacy zero-norm VFS root could trip the same gate before its VFS-init-time cure ever ran. - add()/update() (single and transact()) now normalize an explicit real all-zero vector to the unvectored [] shape before the dimension pin, the ledger flag, and the index ops ever see it (loud, one warn per write, canonical write still succeeds). - The legacy counts.json derivation walk (scanVectoredNounCount) excludes a persisted zero-norm row, matching the live ledger's definition. - A legacy zero-norm VFS root now migrates at open, before the vector-leg gate evaluates, via one O(1) fixed-path read (torn-tolerant — skips rather than aborting init on a torn root, letting the recovery walk heal it) — independent of whether a VirtualFileSystem is ever constructed this session. - update({ id, vector: [] }) (and the same op inside transact()) is now the sanctioned, idempotent unvector door: index removal, exactly-once ledger decrement, no re-embed, and it clears a pending deferred-embed marker rather than leaving it to re-vectorize the row later. The combination with deferEmbedding is a typed refusal. - JsHnswVectorIndex.rebuild() now skips a zero-norm/empty persisted vector when repopulating from canonical (the same belt the live add/replace paths already had), and health()'s index-parity check now compares HNSW size against the vectored-noun ledger rather than the raw metadata-entry count, since a store's VFS root is permanently unvectored by design. Co-Authored-By: Claude Fable 5 --- src/brainy.ts | 351 +++++++++++++-- src/hnsw/hnswIndex.ts | 29 +- src/storage/adapters/fileSystemStorage.ts | 30 +- src/utils/paramValidation.ts | 23 +- tests/integration/vfs-root-zero-norm.test.ts | 31 +- .../zero-norm-unvector-door.test.ts | 399 ++++++++++++++++++ 6 files changed, 810 insertions(+), 53 deletions(-) create mode 100644 tests/integration/zero-norm-unvector-door.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index bfac8c08..ed958a67 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -396,6 +396,15 @@ interface PlannedTransact { * marker outlives its write. */ markerRecords: FactMarkerRecord[] + /** + * Ids the batch's `{ op: 'update' }` unvector door (`vector: []`) needs to + * decrement on the vectored-noun ledger — consumed by `transact()` with a + * proper `await this.storage.noteVectorUnlanded?.(id)` per id, AFTER + * `commitTransaction` resolves (never for a rejected batch). Kept separate + * from `postCommit` (`Array<() => void>`, called synchronously, fire-and- + * forget) because the ledger hook is async and must be awaited. + */ + vectorUnlands: string[] } /** @@ -1504,6 +1513,15 @@ export class Brainy implements BrainyInterface { }).backfillBlobHistoryRefCountsIfNeeded() } + // LEG C (zero-norm/unvector-door law): migrate a legacy zero-norm VFS + // root BEFORE the vector-leg open gate below ever compares the + // canonical vectored-noun count against the vector index's size — see + // migrateLegacyZeroNormVfsRootIfNeeded's JSDoc for why this is a safe + // O(1) exception to "nothing at open may scale with brain size", and + // why it must run here rather than waiting on VirtualFileSystem's own + // (VFS-instance-gated) lazy migration. + await this.migrateLegacyZeroNormVfsRootIfNeeded() + // Rebuild indexes if needed for existing data. Runs to completion before // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true @@ -1518,8 +1536,9 @@ export class Brainy implements BrainyInterface { // cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph // index construction, the eager cold-load, id-resolver + connections- // codec wiring, crash-recovery index rebuild, the replay-gap check, - // legacy VFS blob adoption, blob-history backfill, and the - // rebuildIndexesIfNeeded() gate + migration check. + // legacy VFS blob adoption, blob-history backfill, the legacy + // zero-norm VFS root migration, and the rebuildIndexesIfNeeded() gate + // + migration check. markPhase('index-init-gate') // Register shutdown hooks for graceful count flushing (once globally) @@ -2912,10 +2931,34 @@ export class Brainy implements BrainyInterface { // vector shape, is structurally impossible). The background worker // embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed + let vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) + // THE ZERO-NORM LAW (canonical write side): a zero-norm vector is not a + // vector — it never crosses an engine boundary (the engine pair's seam + // law). This engine's own cosine distance treats an all-zero vector + // safely (a zero-norm operand always scores MAXIMUM distance — see + // isZeroNormVector's JSDoc), but a downstream engine serving squared- + // euclidean distance cannot tell it apart from a legitimate origin + // point — a false attractor that silently darkened 150+ rows in a + // production deployment. The index belt (AddToVectorIndexOperation) + // already refuses to INDEX a zero-norm vector, but until now the + // CANONICAL write still persisted it and the vectored-noun ledger + // counted it — so a near-empty store whose only vectored row was + // zero-norm read "canonical vectored > 0, index size 0" and threw a + // not-ready error at open. Normalize HERE, before the dimension pin, + // the vectored-ledger flag (`SaveNounMetadataOperation`'s `hasVector`), + // and the index ops below ever see it, so it persists as the sanctioned + // "unvectored" `[]` shape instead — the canonical write still succeeds. + if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { + prodLog.warn( + `[Brainy] add(): entity ${id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + vector = [] + } + // Ensure dimensions are set (a deferred-embed stub carries no dimension // information — the worker's real vector goes through the same guard). // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose @@ -3623,25 +3666,53 @@ export class Brainy implements BrainyInterface { // often the host writes. const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged + + // THE ZERO-NORM LAW (canonical write side) — see add()'s matching + // comment: an explicit REAL all-zero vector is not a vector. Normalize + // to the sanctioned "unvectored" `[]` shape BEFORE the dimension + // check, the unvector-door decision below, and the index ops ever see + // it — a local copy; `params.vector` itself is never mutated. + let explicitVector = params.vector + if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { + prodLog.warn( + `[Brainy] update(): entity ${params.id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + explicitVector = [] + } + + // THE SANCTIONED UNVECTOR DOOR: `explicitVector` at length 0 (an + // explicit `vector: []`, or a real all-zero vector just normalized + // above) is an instruction to remove the vector NOW — never "please + // embed". `validateUpdateParams` already refuses combining it with + // `deferEmbedding: true` (an empty array is truthy, so that guard + // fires unconditionally on any explicit `vector`). Idempotent on an + // already-unvectored row: the ledger decrement near the end of this + // method is gated on the PRIOR vector actually having been real. + const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 + // MT5 deferred re-embedding: the OLD vector keeps serving semantic // search — stale-but-present, never absent (the flicker law) — until // the background worker embeds the new data and swaps it atomically. const deferringEmbed = - params.deferEmbedding === true && hasNewData && !params.vector - if (params.vector) { - if (this.dimensions && params.vector.length !== this.dimensions) { + params.deferEmbedding === true && hasNewData && !explicitVector + if (explicitVector) { + // A length-0 explicit vector (the unvector door) carries no + // dimension information — exempt from the check, mirroring add()'s + // own `vector.length > 0` gate on the dimension pin. + if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` ) } - vector = params.vector + vector = explicitVector } else if (hasNewData && !deferringEmbed) { vector = await this.embed(params.data) } // A deferred data change does NOT reindex now (the vector is unchanged; // the worker's atomic swap carries the real reindex later). const needsReindexing = Boolean( - (hasNewData && !deferringEmbed) || params.type || params.vector + (hasNewData && !deferringEmbed) || params.type || explicitVector ) // Always update the noun with new metadata @@ -3735,6 +3806,22 @@ export class Brainy implements BrainyInterface { ? [this.enqueuePendingEmbed(params.id)] : undefined + // Leg D — the unvector door clears a PENDING deferred-embed marker: + // without this, the worker would later embed this row's current data + // and silently re-vector it, defeating the caller's explicit "remove + // the vector now" instruction. The clear rides THIS SAME commit fact + // (an `embed.landed` record with an empty vector — the recovery fold + // disarms a pending marker on ANY `embed.landed` for the id, + // regardless of the vector it carries), so a crash between the write + // and the in-memory clear below still recovers disarmed. Mutually + // exclusive with `embedMarkers` above: `deferringEmbed` requires an + // ABSENT `explicitVector`, so the two branches never both apply. + const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) + const commitRecords: FactMarkerRecord[] | undefined = + embedMarkers ?? (clearsPendingEmbed + ? [{ type: 'embed.landed', id: params.id, vector: [] }] + : undefined) + // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { @@ -3823,7 +3910,33 @@ export class Brainy implements BrainyInterface { } } ] - : undefined, embedMarkers) + : undefined, commitRecords) + + // Leg D continued — the in-memory pending-embed clear runs only AFTER + // the commit above actually succeeded (an aborted update must not + // disarm a marker whose durable `embed.landed` twin was never + // written). + if (clearsPendingEmbed) { + this.clearPendingEmbed(params.id) + prodLog.warn( + `[Brainy] update(): entity ${params.id} had a pending deferred embed — ` + + `the unvector door cleared it ('vector: []' is an explicit instruction, ` + + `never "please embed").` + ) + } + + // Leg D — vectored-ledger decrement for the sanctioned unvector door. + // update()'s own metadata write goes through UpdateNounMetadataOperation + // (isNew=false), so the saveNounMetadata(..., hasVector) seam never + // fires here — noteVectorUnlanded is the ONLY seam, the same + // sanctioned hook unvectorNounForRootMigration() uses. Gated on the + // PRIOR vector having actually been real (non-empty, non-zero-norm): + // an already-unvectored row's second call is a true no-op — no + // decrement, matching the ledger-exactness law (never double-count, + // never drift negative). + if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { + await this.storage.noteVectorUnlanded?.(params.id) + } // Aggregation hook (outside transaction — derived data). `existing` is // the full get() view — every reserved field top-level — and must be @@ -9179,6 +9292,15 @@ export class Brainy implements BrainyInterface { hook() } + // Leg D — vectored-ledger decrements for this batch's unvector-door + // updates (see planTxUpdate's matching comment), applied after the + // commit point and properly awaited (unlike `postCommit`'s synchronous + // fire-and-forget hooks) — each is the same sanctioned hook + // unvectorNounForRootMigration() uses. + for (const id of plan.vectorUnlands) { + await this.storage.noteVectorUnlanded?.(id) + } + // Change feed: the batch's events share its single committed generation. // A rejected batch throws at commitTransaction and never reaches here. this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) @@ -10422,7 +10544,8 @@ export class Brainy implements BrainyInterface { casUpdates: [], createdNouns: new Set(), changeEvents: [], - markerRecords: [] + markerRecords: [], + vectorUnlands: [] } for (const op of ops) { @@ -10544,9 +10667,22 @@ export class Brainy implements BrainyInterface { // marker-less committed row would be a silently missing vector, which is // the disallowed direction). The background worker embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed + let vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) + + // THE ZERO-NORM LAW — see the single-add() insert path's matching + // comment (a zero-norm vector is not a vector; never crosses an engine + // boundary). Normalized here BEFORE the dimension pin and the + // vectored-ledger `hasVector` flag below ever see it. + if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { + prodLog.warn( + `[Brainy] transact add: entity ${id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + vector = [] + } + // Gated on `vector.length > 0` — see the single-add() insert path's // matching comment: an explicit `vector: []` carries no dimension // information either, deferred or not. @@ -10717,17 +10853,62 @@ export class Brainy implements BrainyInterface { const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged let vector = existing.vector - if (params.vector) { - if (this.dimensions && params.vector.length !== this.dimensions) { + + // THE ZERO-NORM LAW + THE SANCTIONED UNVECTOR DOOR — transact() mirror + // of update()'s matching block: an explicit REAL all-zero vector + // normalizes to `[]` (never crosses an engine boundary), and an + // explicit `vector: []` (post-normalization) is the sanctioned unvector + // instruction, exempt from the dimension check. `validateUpdateParams` + // already refuses combining it with `deferEmbedding: true`. + let explicitVector = params.vector + if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { + prodLog.warn( + `[Brainy] transact update: entity ${params.id} was given an explicit all-zero ` + + `vector — a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + explicitVector = [] + } + const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 + + if (explicitVector) { + if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` ) } - vector = params.vector + vector = explicitVector } else if (hasNewData) { vector = await this.embed(params.data) } - const needsReindexing = Boolean(hasNewData || params.type || params.vector) + const needsReindexing = Boolean(hasNewData || params.type || explicitVector) + + // Leg D — the unvector door clears a PENDING deferred-embed marker (see + // update()'s matching comment for the full rationale): the durable + // clear (an `embed.landed` record, empty vector) rides the batch's ONE + // commit fact via `plan.markerRecords`; the in-memory clear is deferred + // to `plan.postCommit` so an aborted batch never disarms a marker whose + // durable twin was never written. + const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) + if (clearsPendingEmbed) { + plan.markerRecords.push({ type: 'embed.landed', id: params.id, vector: [] }) + plan.postCommit.push(() => { + this.clearPendingEmbed(params.id) + prodLog.warn( + `[Brainy] transact update: entity ${params.id} had a pending deferred embed — ` + + `the unvector door cleared it ('vector: []' is an explicit instruction, ` + + `never "please embed").` + ) + }) + } + + // Leg D — vectored-ledger decrement for the sanctioned unvector door, + // deferred to `plan.vectorUnlands` (consumed with a proper `await` in + // `transact()`, AFTER the commit succeeds — see its matching comment). + // Gated on the PRIOR vector having actually been real (non-empty, + // non-zero-norm): idempotent on an already-unvectored row. + if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { + plan.vectorUnlands.push(params.id) + } const newMetadata = params.merge !== false @@ -12431,21 +12612,49 @@ export class Brainy implements BrainyInterface { const metadataStats = await this.metadataIndex.getStats() const graphSize = await this.graphIndex.size() - // 1. Index size parity. HNSW must hold at least one node per indexed entity. - if (hnswSize === metadataStats.totalEntries) { + // 1. Index size parity. HNSW must hold one node per VECTORED noun — the + // vectored-noun ledger (`getCanonicalCounts().vectors.all`), NOT the raw + // metadata-entry count: every store's VFS root is PERMANENTLY unvectored + // (`vector: []` by design — a zero-norm/empty vector never crosses into + // the index, see AddToVectorIndexOperation/JsHnswVectorIndex.rebuild()'s + // matching belts), and a not-yet-landed deferred embed is unvectored + // too. Comparing against total entries counted the always-unvectored + // root as a permanent 1-node "drift" on every VFS-having store — a false + // warn on an otherwise perfectly healthy handoff. `vectors.all` is + // already the documented coverage denominator for exactly this + // comparison (see `CanonicalCounts.vectors`'s JSDoc). Falls back to the + // metadata-entry count when the ledger is unavailable or suspect (a + // storage adapter without the optional hook, or an unrecounted store) — + // never worse than the prior behavior in that case. + const vectorLedgerForParity = await this.storage.getCanonicalCounts?.() + const vectorParityTarget = + vectorLedgerForParity && !vectorLedgerForParity.suspect + ? vectorLedgerForParity.vectors.all + : metadataStats.totalEntries + if (hnswSize === vectorParityTarget) { checks.push({ name: 'index-parity', status: 'pass', - message: `HNSW (${hnswSize}) and metadata index (${metadataStats.totalEntries}) agree.`, - details: { hnswSize, metadataEntries: metadataStats.totalEntries, graphRelationships: graphSize } + message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) agree.`, + details: { + hnswSize, + vectoredNouns: vectorParityTarget, + metadataEntries: metadataStats.totalEntries, + graphRelationships: graphSize + } }) } else { - const drift = Math.abs(hnswSize - metadataStats.totalEntries) + const drift = Math.abs(hnswSize - vectorParityTarget) checks.push({ name: 'index-parity', - status: drift > Math.max(10, metadataStats.totalEntries * 0.01) ? 'fail' : 'warn', - message: `HNSW (${hnswSize}) and metadata (${metadataStats.totalEntries}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, - details: { hnswSize, metadataEntries: metadataStats.totalEntries, drift } + status: drift > Math.max(10, vectorParityTarget * 0.01) ? 'fail' : 'warn', + message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, + details: { + hnswSize, + vectoredNouns: vectorParityTarget, + metadataEntries: metadataStats.totalEntries, + drift + } }) } @@ -16066,6 +16275,79 @@ export class Brainy implements BrainyInterface { return !this.pluginRegistry.hasProvider('embeddings') } + /** + * @description LEG C of the zero-norm/unvector-door law — migrate a + * legacy zero-norm VFS root BEFORE the vector-leg open gate + * ({@link rebuildIndexesIfNeeded}'s `vectorCoverageGap` check) ever + * compares the canonical vectored-noun count against the vector index's + * size. A pre-fix store may have persisted the VFS root (the fixed + * all-zeros UUID) with a REAL all-zero placeholder vector — lawful inside + * brainy (`cosineDistance` treats a zero-norm operand as MAXIMUM distance, + * see {@link isZeroNormVector}'s JSDoc) but never indexed (the index belt + * refuses to insert a zero-norm vector) and never meant to cross an + * engine boundary. Left unmigrated, the canonical ledger still counts it + * as vectored while the vector index correctly holds nothing for it — a + * near-empty store whose ONLY vectored row is this zero-norm root reads + * "canonical vectored 1, index size 0" and throws + * `VectorIndexNotReadyError` at open, going DARK instead of serving. + * + * THE LIFECYCLE LAW: nothing at open may scale with brain size. This step + * is safe under that law BECAUSE the VFS root lives at a FIXED, + * well-known id (`00000000-0000-0000-0000-000000000000` — mirrors + * `VirtualFileSystem.VFS_ROOT_ID`; kept as a literal here, the same + * convention as the other reserved-root literals in this file and in + * `db/factLog.ts`/`db/portableGraph.ts` — `brainy.ts` cannot import + * `VirtualFileSystem.ts`, which itself imports `Brainy`) — this is ONE + * direct canonical read by id (`storage.getNoun`, the same O(1) + * fixed-path lookup {@link unvectorNounForRootMigration} itself uses + * internally), NEVER a listing or a walk over `entities/nouns/**`. An + * absent root (a store that has never used the VFS) is a no-op, no error. + * + * Runs UNCONDITIONALLY at every open, independent of whether a + * `VirtualFileSystem` is ever constructed this session — the vector-leg + * gate this fixes runs during Brainy's OWN init, before any + * `VirtualFileSystem` instance exists to run its own lazy migration at + * `doInitializeRoot()` (kept in place as the second line of defense for a + * VFS actually opened this session — belt AND suspenders, never either + * alone). + */ + private async migrateLegacyZeroNormVfsRootIfNeeded(): Promise { + const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + // TORN-TOLERANT: a torn root record is a recovery-walk healer's job + // (see tests/integration/recovery-walk-tolerance.test.ts — an init-time + // walk that meets a torn record narrates+counts, via the adapter's own + // loud floor at the read site, and heals PAST it; the open itself must + // still succeed), not this O(1) migration check's. Skip this open's + // migration attempt rather than aborting init(): this leg is a + // defensive EXTRA (the index belt + VirtualFileSystem's own + // doInitializeRoot() migration still stand as the other lines of + // defense), and it retries harmlessly at a later open once the root + // heals. + let root: HNSWNounWithMetadata | null + try { + root = await this.storage.getNoun(VFS_ROOT_ID) + } catch (err) { + if ((err as { code?: string }).code !== 'TORN_RECORD') throw err + prodLog.warn( + `[Brainy] open(): the VFS root's record is TORN — skipping the zero-norm root ` + + `migration check this open (the recovery walk is the healer; this migration ` + + `retries harmlessly once the root heals).` + ) + return + } + if (!root || !Array.isArray(root.vector) || root.vector.length === 0) return + if (!isZeroNormVector(root.vector)) return + const migrated = await this.unvectorNounForRootMigration(VFS_ROOT_ID) + if (migrated) { + prodLog.warn( + `[Brainy] open(): migrated the VFS root's legacy all-zero placeholder vector to ` + + `the unvectored shape (zero-norm vectors never cross an engine boundary) — run ` + + `before the vector-leg open gate compares canonical-vectored-count against the ` + + `vector index, so a near-empty store never reads a false coverage gap.` + ) + } + } + /** * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's * persisted vector from a real (non-empty) vector to the "unvectored" @@ -16075,13 +16357,18 @@ export class Brainy implements BrainyInterface { * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the * coverage ledger never silently drifts. * - * Exists SOLELY for the VFS root zero-norm migration (see - * `VirtualFileSystem.doInitializeRoot()`, which detects a persisted root - * whose vector is the legacy all-zero placeholder and calls this once per - * store). This is NOT a general-purpose "clear my vector" API — ordinary - * application data has no sanctioned path from vectored back to - * unvectored (`update()` refuses an empty vector as a dimension mismatch, - * by design). Never call this outside the VFS root migration. + * Exists SOLELY for the VFS root zero-norm migration, called from two + * sites that detect the same legacy shape (a persisted root whose vector + * is the legacy all-zero placeholder): {@link migrateLegacyZeroNormVfsRootIfNeeded} + * (this brain's own init sequence, BEFORE the vector-leg open gate — Leg + * C of the zero-norm/unvector-door law) and + * `VirtualFileSystem.doInitializeRoot()` (the second line of defense, for + * a VFS actually constructed this session). This is NOT the general- + * purpose unvector API — ordinary application data uses the sanctioned + * unvector DOOR instead (`update({ id, vector: [] })` / the same op inside + * `transact()`), which decrements the ledger and clears any pending + * deferred-embed marker inline; it does not call this method. Never call + * this outside a VFS root migration. * * Idempotent: a noun already unvectored (`vector.length === 0`) or absent * is a no-op — safe to call on every `init()`. diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 77e4f84d..1ed8f9ea 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -10,7 +10,7 @@ import { Vector, VectorDocument } from '../coreTypes.js' -import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' +import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' @@ -1768,6 +1768,33 @@ export class JsHnswVectorIndex implements VectorIndexProvider { // Process all nouns at once for (const nounData of result.items) { try { + // THE ZERO-NORM LAW / THE INDEX BELT — bulk-rebuild leg: a row + // holding no REAL vector (the "unvectored" `[]` shape — a + // deferred embed's stub, or a row the sanctioned unvector door + // rewrote) must never enter the index, mirroring the guard + // AddToVectorIndexOperation/ReplaceInVectorIndexOperation already + // enforce on the live transactional write paths. This matters + // HERE specifically: a row can be unvectored (canonical vector + // rewritten to `[]`) while its PERSISTED HNSW graph metadata + // (`getVectorIndexData` — level/connections) is still present + // from before the unvector, so `hnswData`'s mere presence below + // is not proof the row belongs in the index — only the + // canonical vector itself is authoritative. Checked against + // `nounData.vector` (the FULL loaded vector), never the + // to-be-truncated `noun.vector` below, so this holds regardless + // of the eager/lazy preload strategy chosen further down. + if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) { + continue + } + if (isZeroNormVector(nounData.vector)) { + prodLog.warn( + `[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` + + `zero-norm (a zero-norm vector is not a vector and never crosses an ` + + `engine boundary)` + ) + continue + } + // Load HNSW graph data for this entity const hnswData = await this.storage.getVectorIndexData(nounData.id) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 817e5e36..4f2a43b0 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -19,6 +19,7 @@ import { import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' import { prodLog } from '../../utils/logger.js' +import { isZeroNormVector } from '../../utils/distance.js' import { TornRecordError, isUnparseablePayloadError, @@ -2841,14 +2842,20 @@ export class FileSystemStorage extends BaseStorage { } /** - * Count canonical nouns holding a REAL (non-empty) vector — the vectored- - * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence - * cannot be decided from the id-directory listing alone: a deferred-embed - * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`) - * until its embed LANDS, so this walk reads every noun's `vectors.json` - * CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time - * legacy-counts.json derivation or a lost/corrupted counts.json recovery; - * the result is persisted so this scan never repeats. + * Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector — + * the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, + * presence cannot be decided from the id-directory listing alone: a + * deferred-embed noun's `vectors.json` EXISTS (written at `add()` time + * with `vector: []`) until its embed LANDS, so this walk reads every + * noun's `vectors.json` CONTENT — O(nouns) reads, not O(ids) listing. + * ZERO-NORM LAW: a real all-zero vector is not a vector — it never counts + * here either (Brainy's write paths normalize an explicit zero-norm + * vector to `[]` at write time, but a store created before that fix may + * still carry legacy all-zero rows on disk; this derivation must agree + * with the live ledger's definition of "vectored" regardless of when the + * row was written). Used ONLY for a one-time legacy-counts.json derivation + * or a lost/corrupted counts.json recovery; the result is persisted so + * this scan never repeats. */ private async scanVectoredNounCount(): Promise { const base = path.join(this.rootDir, 'entities', 'nouns') @@ -2862,7 +2869,12 @@ export class FileSystemStorage extends BaseStorage { for (const entry of ids) { if (!entry.isDirectory()) continue const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) - if (record && Array.isArray(record.vector) && record.vector.length > 0) { + if ( + record && + Array.isArray(record.vector) && + record.vector.length > 0 && + !isZeroNormVector(record.vector) + ) { vectored++ } } diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 606516ac..00790a4a 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -613,6 +613,17 @@ export function validateUpdateParams(params: UpdateParams): void { // null/undefined means "no new data was given". const hasData = params.data !== undefined && params.data !== null if ((params as UpdateParams & { deferEmbedding?: boolean }).deferEmbedding === true) { + if (params.vector && params.vector.length === 0) { + // The nonsensical combination Leg D of the zero-norm/unvector-door law + // refuses: `vector: []` is the SANCTIONED UNVECTOR DOOR — an explicit + // instruction to remove the vector NOW, never "please embed" — so it + // cannot be paired with a request to defer an embed. + throw new Error( + `update(): 'vector: []' (the unvector door) cannot be combined with ` + + `'deferEmbedding: true' — an unvector is an explicit instruction to remove ` + + `the vector now, not a request to defer an embed. Drop one of the two.` + ) + } if (params.vector) { throw new Error( `update(): deferEmbedding cannot be combined with an explicit 'vector' — ` + @@ -649,8 +660,16 @@ export function validateUpdateParams(params: UpdateParams): void { throw new Error(`invalid NounType: ${params.type}`) } - // Validate vector dimensions if provided - if (params.vector) { + // Validate vector dimensions if provided. A length-0 vector is the + // SANCTIONED UNVECTOR DOOR (see brainy.ts update()'s matching comment): an + // explicit `vector: []` — or a real all-zero vector, normalized to `[]` + // upstream by the zero-norm law — carries no dimension information, + // exactly like validateAddParams's identical exemption, so it is exempt + // from the dimension check rather than refused as a "0-dimensional + // vector". (The `deferEmbedding` combination above already refuses + // `vector: []` paired with `deferEmbedding: true` — an empty array is + // truthy, so that guard fires unconditionally on any explicit `vector`.) + if (params.vector && params.vector.length > 0) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) diff --git a/tests/integration/vfs-root-zero-norm.test.ts b/tests/integration/vfs-root-zero-norm.test.ts index 00260eb4..577ae7ee 100644 --- a/tests/integration/vfs-root-zero-norm.test.ts +++ b/tests/integration/vfs-root-zero-norm.test.ts @@ -18,9 +18,16 @@ * harness reproduces exactly what a pre-fix store looked like on disk) * is rewritten to `[]` on the next `init()`, the ledger is decremented * through the sanctioned path, and a second `init()` is a no-op. - * (c) THE BELT at the live provider-write seam: an entity added with an - * EXPLICIT all-zero vector (any dimension) still lands its canonical - * write, but the vector-index insert is refused loudly. + * (c) THE CANONICAL-WRITE NORMALIZATION (Leg A of the follow-up + * zero-norm/unvector-door fix): an entity added with an EXPLICIT + * all-zero vector (any dimension) is normalized to the "unvectored" + * `[]` shape BEFORE the canonical write, the ledger flag, and the index + * ops ever see it — the canonical write still succeeds, loudly, and the + * vector-index insert never happens (nothing to index). Supersedes the + * original "canonical keeps the zero vector, only the index refuses" + * shape: a downstream engine's health-report gate reads the canonical + * ledger directly, so leaving a zero-norm vector on the canonical side + * re-opened the exact false-attractor risk this whole fix closes. * (d) the migrated root never surfaces in `find()` results (it was already * hidden behind `visibility: 'system'` — this pin holds regardless). */ @@ -142,7 +149,7 @@ describe('VFS root zero-norm cure', () => { await brain.close() }) - it('(c) the live-write belt: an entity added with an explicit all-zero vector lands its canonical write, but the vector-index insert is refused loudly', async () => { + it('(c) canonical-write normalization: an entity added with an explicit all-zero vector persists UNVECTORED ([]), loudly, and never reaches the vector index', async () => { const dir = mkTmp() const brain = openBrain(dir) await brain.init() @@ -150,20 +157,26 @@ describe('VFS root zero-norm cure', () => { const warnSpy = vi.spyOn(prodLog, 'warn') const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const ledgerBefore = await brain.storage.getCanonicalCounts() const zeroVector = new Array(384).fill(0) const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) - // The canonical write succeeded — the entity is fully readable with its - // (real, all-zero) vector intact. + // The canonical write succeeded — but the zero-norm vector was + // normalized to the "unvectored" `[]` shape BEFORE it was persisted + // (Leg A: a zero-norm vector is not a vector — it never crosses an + // engine boundary, canonical side included). const entity = await brain.get(id, { includeVectors: true }) expect(entity).not.toBeNull() - expect(entity.vector).toEqual(zeroVector) + expect(entity.vector).toEqual([]) - // The vector-index insert was skipped — the index size never moved. + // Nothing to index — the vector-index size never moved, and the + // vectored-noun ledger never counted this row. const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size expect(sizeAfter).toBe(sizeBefore) + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - // The refusal was LOUD and named the entity. + // The normalization was LOUD and named the entity. const loudCall = warnSpy.mock.calls.find( (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') ) diff --git a/tests/integration/zero-norm-unvector-door.test.ts b/tests/integration/zero-norm-unvector-door.test.ts new file mode 100644 index 00000000..d3290010 --- /dev/null +++ b/tests/integration/zero-norm-unvector-door.test.ts @@ -0,0 +1,399 @@ +/** + * @module tests/integration/zero-norm-unvector-door + * @description THE SEAM LAW, GENERALIZED: "a zero-norm vector is not a + * vector — it never crosses an engine boundary." `tests/integration/ + * vfs-root-zero-norm.test.ts` pins the VFS-root-specific cure; this file + * pins the follow-up that generalizes it to every write path plus the + * sanctioned door for shedding a vector on purpose. + * + * Four legs pinned here: + * (A) THE CANONICAL WRITE NORMALIZES ZERO-NORM TO `[]` — `add()` (single and + * `transact()`) persists an explicit real all-zero vector as the + * "unvectored" `[]` shape, loudly, before the ledger flag/dimension + * pin/index ops ever see it. The canonical write still succeeds. + * (B) THE LEGACY DERIVATION IS ZERO-NORM-AWARE — a lost/corrupted + * `counts.json`'s one-time re-derivation walk excludes a persisted + * zero-norm row from the vectored-noun scalar, matching the live + * ledger's definition of "vectored". + * (C) THE LEGACY VFS ROOT MIGRATES AT OPEN, BEFORE THE GATE, IN O(1) — a + * store whose ONLY vectored row is a legacy all-zero VFS root opens + * clean (no `VectorIndexNotReadyError`), via one fixed-path read, never + * a listing. + * (D) THE UNVECTOR DOOR — `update({ id, vector: [] })` (and the same op + * inside `transact()`) is the sanctioned, idempotent way to shed a + * vector on purpose: ledger decrement exactly once, index removal, no + * re-embed, and a pending deferred-embed marker is cleared rather than + * left to re-vectorize the row later. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { prodLog } from '../../src/utils/logger.js' +import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' +import { BaseStorage } from '../../src/storage/baseStorage.js' + +const ROOT_ID = '00000000-0000-0000-0000-000000000000' + +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-zero-norm-unvector-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + vi.restoreAllMocks() + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) +} + +const countsPath = (root: string) => path.join(root, '_system', 'counts.json') + +describe('zero-norm canonical write + the sanctioned unvector door', () => { + it('(A1) add() with an explicit all-zero vector persists [], warns loudly, never indexes, and the ledger is unchanged', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const warnSpy = vi.spyOn(prodLog, 'warn') + + const zeroVector = new Array(384).fill(0) + const id = await brain.add({ data: 'zero-norm add', type: NounType.Document, vector: zeroVector }) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore) + + const loud = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') + ) + expect(loud).toBeDefined() + + await brain.close() + }) + + it('(A2) transact() add with an explicit all-zero vector — the same canonical normalization', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const warnSpy = vi.spyOn(prodLog, 'warn') + const zeroVector = new Array(384).fill(0) + const id = 'aaaaaaaa-0000-4000-8000-000000000001' + + await brain.transact([ + { op: 'add', id, type: NounType.Document, data: 'zero-norm transact add', vector: zeroVector } + ]) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + const loud = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') + ) + expect(loud).toBeDefined() + + await brain.close() + }) + + it('(B) the legacy counts.json derivation excludes a persisted zero-norm row from the vectored-noun scalar', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() + + // The VFS root alone (unvectored — []) — the floor. + const baseline = (await brain.storage.getCanonicalCounts()).vectors.all + + const realId = await brain.add({ data: 'a real vectored document', type: NounType.Document }) + + // Plant the legacy all-zero shape BY HAND: a genuine identity record + // (via add(), so it has real metadata) whose vector leg is then + // overwritten directly through the raw storage primitive — bypassing + // Leg A's canonical-write normalization entirely (brain.storage.saveNoun + // is not Brainy.add()/update()'s normalized path) — reproducing exactly + // what a pre-fix store could have persisted on disk. + const zeroId = await brain.add({ data: 'a legacy zero-norm document', type: NounType.Document }) + const zeroVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: zeroId, vector: zeroVector, connections: new Map(), level: 0 }) + + await brain.flush() + await brain.close() + + // Remove counts.json so the next open re-derives from scratch (the + // one-time legacy/lost-file derivation path — Leg B). + fs.rmSync(countsPath(dir), { force: true }) + + brain = openBrain(dir) + await brain.init() + const ledger = await brain.storage.getCanonicalCounts() + // Only realId counts; zeroId's persisted all-zero vector does not. + expect(ledger.vectors.all).toBe(baseline + 1) + + await brain.close() + }) + + it('(C) a legacy all-zero VFS root as the ONLY vectored row: open succeeds with no not-ready error, via an O(1) fixed-path read (no entities-tree readdir), and the ledger is 0 after open', async () => { + const dir = mkTmp() + + // SESSION 1 — build the legacy shape: the root is a REAL all-zero + // 384-dim vector, genuinely indexed and genuinely ledgered — exactly + // what a pre-fix store's root looked like on disk (see + // vfs-root-zero-norm.test.ts pin (b) for the identical harness). + // `index.addItem` is called directly (bypassing the transactional + // zero-norm belt) because the pre-fix code path had no such belt — this + // harness must match history, not the cure. No other entity is added, + // so the root is the store's ONLY vectored row. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.flush() + expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(1) + await brain.close() + + // SESSION 2 — reopen with a FAKE native vector provider that claims + // `serving: true` at `size()===0` (the exact shape a downstream + // engine's own health report can legitimately carry — same technique as + // tests/integration/vector-leg-open-build.test.ts). This is the ONLY + // codepath where the vector-leg open gate's FAIL-TYPED throw + // (VectorIndexNotReadyError) can fire; the built-in JS engine alone + // never reaches it (the size-heuristic branch just rebuilds instead) — + // so this is the faithful reproduction of the incident Leg C closes. + const readdirCalls: string[] = [] + const originalReaddir = fs.promises.readdir.bind(fs.promises) + vi.spyOn(fs.promises, 'readdir').mockImplementation(((...args: any[]) => { + readdirCalls.push(String(args[0])) + return (originalReaddir as any)(...args) + }) as any) + + // Spy at the PROTOTYPE level (BaseStorage.getNoun) — the new brain's + // storage instance does not exist until init() runs, so an + // instance-level spy cannot be installed beforehand. Records the + // readdir-call delta across the FIRST call made with the root id — + // Leg C's own fixed-path read — proving it needs no directory listing. + let readdirDeltaDuringRootRead: number | null = null + const originalGetNoun = BaseStorage.prototype.getNoun + vi.spyOn(BaseStorage.prototype, 'getNoun').mockImplementation(async function ( + this: unknown, + id: string + ) { + const before = readdirCalls.length + const result = await originalGetNoun.call(this as BaseStorage, id) + if (id === ROOT_ID && readdirDeltaDuringRootRead === null) { + readdirDeltaDuringRootRead = readdirCalls.length - before + } + return result + }) + + brain = openBrain(dir) + brain.use({ + name: 'fake-native-vector-unledgered-coverage', + activate: async (ctx: any) => { + ctx.registerProvider('vector', (config: any, distance: any, options: any) => { + const real = new JsHnswVectorIndex(config, distance, options) + let rebuilt = false + const originalRebuild = real.rebuild.bind(real) + ;(real as any).rebuild = async (...args: any[]) => { + const r = await originalRebuild(...args) + rebuilt = true + return r + } + const originalSize = real.size.bind(real) + ;(real as any).size = () => (rebuilt ? originalSize() : 0) + ;(real as any).healthReport = () => ({ + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 0, + generation: 1, + unledgered: ['vector-coverage'] + }) + return real + }) + return true + } + }) + + // Must NOT throw VectorIndexNotReadyError (or anything else) — a + // near-empty store whose only vectored row is the zero-norm root must + // never go dark. + await brain.init() + + const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) + expect(migratedRoot.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(0) + + expect(readdirDeltaDuringRootRead).toBe(0) + + await brain.close() + }) + + describe('the sanctioned unvector door', () => { + it('(D1) update({ id, vector: [] }) unvectors a real vectored row — canonical [], removed from the index, ledger decremented by exactly 1, no embed call', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + await brain.flush() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + + const embedSpy = vi.spyOn(brain, 'embed') + await brain.update({ id, vector: [] }) + expect(embedSpy).not.toHaveBeenCalled() + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore - 1) + + await brain.close() + }) + + it('(D2) idempotent: a second update({ id, vector: [] }) on an already-unvectored row is a true no-op — no error, no further decrement', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + await brain.flush() + + await brain.update({ id, vector: [] }) + const ledgerAfterFirst = await brain.storage.getCanonicalCounts() + + await brain.update({ id, vector: [] }) + const ledgerAfterSecond = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfterFirst.vectors.all) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + await brain.close() + }) + + it('(D3) a PENDING deferred-embed row: the unvector door clears the marker; awaitPendingEmbeds() then leaves it unvectored', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + // Prevent the background worker from ever actually running — it is + // fire-and-forget from add(), and a real run would race this test's + // own assertions (see tests/integration/vector-leg-open-build.test.ts + // for the same concern). This isolates exactly the marker-clearing + // behavior under test. + vi.spyOn(brain as any, 'kickEmbedWorker').mockImplementation(() => {}) + + const id = await brain.add({ + data: 'deferred content, never embedded', + type: NounType.Document, + deferEmbedding: true + }) + expect(brain.pendingEmbedCount()).toBe(1) + + const warnSpy = vi.spyOn(prodLog, 'warn') + await brain.update({ id, vector: [] }) + + expect(brain.pendingEmbedCount()).toBe(0) + const clearedWarn = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('pending') + ) + expect(clearedWarn).toBeDefined() + + // The barrier must not hang and must not re-vectorize the row — the + // worker (still mocked to a no-op) never runs again. + await brain.awaitPendingEmbeds() + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + await brain.close() + }) + + it('(D4) update({ vector: [], deferEmbedding: true }) is a typed refusal — the unvector door cannot be paired with a deferred embed', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + const before = await brain.get(id, { includeVectors: true }) + + await expect( + brain.update({ id, vector: [], deferEmbedding: true }) + ).rejects.toThrow(/unvector door/i) + + // Refused before any write — the row is untouched. + const after = await brain.get(id, { includeVectors: true }) + expect(after.vector).toEqual(before.vector) + + await brain.close() + }) + + it('(D5) the transact() twin of the unvector door decrements the ledger exactly once, and is idempotent on a second call', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document for transact unvector', type: NounType.Document }) + await brain.flush() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + + await brain.transact([{ op: 'update', id, vector: [] }]) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore - 1) + + // Idempotent through transact() too. + await brain.transact([{ op: 'update', id, vector: [] }]) + const ledgerAfterSecond = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfter.vectors.all) + + await brain.close() + }) + }) +}) From 6f93108648bf15e90958bbe02f7c7857c900d387 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 14:47:48 -0700 Subject: [PATCH 036/150] chore(release): 10.4.2-rc.1 --- CHANGELOG.md | 17 +++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec925642..0b154af5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) + +- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) +- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) +- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1) +- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4) +- Merge branch 'next/enumeration-identity-rekey' (204d74c1) +- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16) +- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a) +- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab) +- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9) +- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245) +- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee) +- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb) +- chore(release): 10.4.1-rc.1 (7870dc40) + + ### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) diff --git a/package-lock.json b/package-lock.json index 70635835..889c67c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 4125b073..214feeac 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From a082e0efdd502941eb1baa2fa3d1ac30b0d0b095 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 15:07:25 -0700 Subject: [PATCH 037/150] docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy --- RELEASES.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index bc400412..6de11244 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,95 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.4.2 — 2026-08-28 (a zero-norm vector is not a vector) + +**This is the last release of the MIT engine under the `@soulcraft/brainy` name.** +The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API, +client library, types and protocol, an openly specified canonical format, and the TypeScript +reference engine, scoped honestly as a single-node engine for stores up to roughly one +million rows. The `@soulcraft/brainy` name passes to the native engine, **Brainy**, at a +major version bump; that engine implements the same API over the same open format at +production scale, requires a license, and refuses loudly without one. Nothing changes +for existing installs until that major ships; the move is announced with it. + +Six fixes, one law: a vector with no magnitude carries no information, so it must +never reach a vector index — in any engine — and the canonical store must say so. + +- **The permanently-unvectored row.** `add({ ..., vector: [] })` (and the same item + shape in `addMany` / `transact`) is now the sanctioned "no vector" row: persisted + with an empty vector leg, never embedded, never indexed, counted as unvectored in + the canonical ledger. Metadata-only rows — telemetry tallies, counters, plumbing — + no longer need a placeholder vector and never enter the vector leg. `vector: []` + together with `deferEmbedding: true` is refused with a typed error (a supplied + vector has nothing to defer). Previously `vector: []` threw a dimension error. +- **The unvector door.** `update({ id, vector: [] })` (and its `transact()` twin) is + the sanctioned way to strip a vector from an existing row: canonical vector → `[]`, + removal from the vector index, the vectored ledger decremented exactly once — and + idempotent, so a resumed cleanup pass may simply re-issue. It never re-embeds, and + it clears a pending deferred-embed marker durably so the background worker cannot + re-vector the row later. Note that a rebuild never sheds vectors (it re-derives the + index from canonical rows); shedding historical vectors needs this door. +- **Zero-norm vectors are normalized at the write.** An explicit all-zero vector on + any write path is persisted as unvectored (`[]`) with one warning naming the row; + the vector-index operations keep their own refusal as a second line. The engine's + own VFS root, which used to persist a deliberate all-zero placeholder (harmless + under cosine distance, a false attractor under a downstream engine's + squared-euclidean serving — a production incident this week), is now created + unvectored, and an existing store's legacy root is migrated on open by a single + fixed-path read before the health gate runs — never a walk. +- **Enumeration keys on the identity record.** `getNouns()` / `getVerbs()` and the + cursor walks behind them enumerate by the metadata record, the same key the + canonical ledger counts by — previously the walk keyed on the vector file, so a + row holding metadata but no vector was counted yet never yielded (a permanent + "missing" phantom in coverage math), while an orphaned vector-only directory + could be yielded as a phantom id. The recovery fold also never deletes an existing + vector when it replays a metadata-only after-image (preserve-if-absent). One + documented gap remains: a verb's endpoints live only in its vector leg, so a + metadata-only verb is counted and loudly skipped, never fabricated — the fix is a + canonical-format change and lands with the open format. +- **The ledger's one-time derivation counts identity records.** Stores upgraded from + pre-ledger versions derived their ALL-visibility scalars once by counting id + directories, which included ghost and scar containers left by an old partial-delete + defect — an inflated denominator whose coverage row could never reach exact. The + derivation now counts only directories holding a metadata record, `counts.json` + carries a derivation-rule stamp, and a ledger derived under the old rule is marked + `suspect` at open (one O(1) field read, one warning) so the online `repairIndex()` + path clears it with a real recount. +- **The vector index refuses what it cannot hold.** `rebuild()` skips unvectored and + zero-norm rows (one summary line), re-pins the vector dimension from the first real + vector after a restart (previously a restart left the pin unset, so a wrong-length + insert became the new pin instead of being rejected), and `addItem` / `updateItem` + throw a typed `EmptyVectorIndexError` on a length-0 vector instead of ever storing + a vector-less node. +- **Smaller:** a failing plugin activation now rethrows with the original error as + `cause` (the originating file and line survive to the caller's log); build + generators stamp from the repository history of their inputs instead of wall clock, + so two builds of the same tree are byte-identical. + +Adoption: one restart, paired with its native-engine release. The first open of an +existing store runs the legacy-root migration (one narrated line) and, on stores that +upgraded from pre-ledger versions, marks the ledger suspect until the next sanctioned +recount — no rebuild in either case. + +## v10.4.1 — 2026-08-26 (reads refuse per family; an unchanged write never re-embeds) + +Two production defects from the same week, fixed together as a patch to 10.4.0. + +- **The read gate is per family.** A read now refuses only when the index family it + actually consults is unhealthy: a metadata filter is served while the vector leg is + rebuilding; a semantic query is refused only by the vector family; a graph + traversal only by the graph family. Previously any unhealthy family refused every + read on the brain — under a long vector rebuild, a production deployment's + metadata-only reads were refused for the duration, and the retries became a write + pump of their own. +- **Unchanged data never re-embeds.** `update()` compares the incoming `data` + structurally with the stored record; an update carrying identical data (a common + shape for periodic upserts) no longer embeds again and no longer churns the vector + leg. Previously every such update re-embedded and re-inserted, which under load + saturated the vector index with near-identical vectors. + +Adoption: one restart, paired with its native-engine release. + ## v10.4.0 — 2026-08-25 (the health report has a name) Three related cures, one root cause: an index deciding whether it could be trusted From 1f34fc6ce496ef9a359be04153da1397191f50b3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 15:14:03 -0700 Subject: [PATCH 038/150] chore(release): 10.4.2 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b154af5..d13a2d66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) + +- docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) + + ### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) - Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) diff --git a/package-lock.json b/package-lock.json index 889c67c8..fb88681f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.2", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 214feeac..17983c19 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 9f248b24952ac061fc05844a57541c8562ba35f1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 16:56:45 -0700 Subject: [PATCH 039/150] =?UTF-8?q?docs(releases):=2010.4.3=20=E2=80=94=20?= =?UTF-8?q?Open=20Brainy's=20first=20release=20under=20the=20new=20name,?= =?UTF-8?q?=20same=20engine=20as=2010.4.2;=20The=20Source=20is=20the=20one?= =?UTF-8?q?=20registry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index 6de11244..b89beba0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,7 +31,30 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- -## v10.4.2 — 2026-08-28 (a zero-norm vector is not a vector) +## v10.4.3 — 2026-08-27 (Open Brainy's first release) + +**`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for +byte — only the name, the registry, and the pointers changed.** Install: + +```bash +npm install @soulcraftlabs/brainy +``` + +with the registry line in your `.npmrc` (anonymous read): + +``` +@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/ +``` + +- **The Source is the one registry.** Open Brainy publishes to source.soulcraft.com only; the + npmjs republish step is retired from the release rail. Existing npmjs versions of + `@soulcraft/brainy` stay as they are and receive no new versions. +- **The repository moved** to `soulcraftlabs/open-brainy` on The Source; the old path redirects. +- **No engine change.** Everything in the 10.4.2 notes applies unchanged; adoption is one + install-line change (`@soulcraft/brainy` → `@soulcraftlabs/brainy`), which downstream + applications make together with their native-engine bump. + +## v10.4.2 — 2026-08-27 (a zero-norm vector is not a vector) **This is the last release of the MIT engine under the `@soulcraft/brainy` name.** The MIT package continues as **Open Brainy** — `@soulcraftlabs/brainy`: the open API, From a99b1e83c4548d6dec78ee1840267678a1d68dc1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 17:07:09 -0700 Subject: [PATCH 040/150] chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepares the repo for its new home at soulcraftlabs/open-brainy ahead of the Forgejo transfer: package name, publish registry, release script, and every install/import reference across docs, src, tests, examples, and integrations now point at @soulcraftlabs/brainy on The Source. The npmjs storefront leg and byte-identity pair verification are stripped from the release script — The Source is now the only publish target. README gains an Open Brainy explainer and a registry note for consumers. @soulcraft/brainy 10.4.2 was the last release under the old name. --- .claude/skills/architecture.md | 2 +- .forgejo/workflows/publish-source.yml | 14 +-- CLAUDE.md | 4 +- README.md | 16 ++-- SECURITY.md | 2 +- bin/brainy-ts.js | 2 +- bun.lock | 2 +- docs/DEVELOPER_LEARNING_PATH.md | 12 +-- docs/FIND_SYSTEM.md | 2 +- docs/MIGRATION-V3-TO-V4.md | 12 +-- docs/PLUGINS.md | 24 ++--- docs/PRODUCTION_SERVICE_ARCHITECTURE.md | 4 +- docs/README.md | 2 +- docs/RELEASE-GUIDE.md | 2 +- docs/SCALING.md | 4 +- docs/api/README.md | 10 +- .../architecture/data-storage-architecture.md | 2 +- docs/architecture/finite-type-system.md | 6 +- .../multiprocess-storage-mixin.md | 2 +- docs/architecture/noun-verb-taxonomy.md | 4 +- docs/architecture/zero-config.md | 2 +- docs/concepts/field-addressing.md | 2 +- docs/concepts/index-health.md | 2 +- docs/concepts/storage-adapters.md | 16 ++-- docs/guides/aggregation.md | 2 +- docs/guides/framework-integration.md | 24 ++--- docs/guides/import-anything.md | 2 +- docs/guides/import-progress-examples.md | 2 +- docs/guides/import-quick-reference.md | 4 +- docs/guides/inspection.md | 2 +- docs/guides/installation.md | 14 +-- docs/guides/migration-3.36.0.md | 10 +- docs/guides/model-loading.md | 2 +- docs/guides/namespace-migration.md | 4 +- docs/guides/nextjs-integration.md | 18 ++-- docs/guides/optimistic-concurrency.md | 4 +- docs/guides/quick-start.md | 6 +- docs/guides/standard-import-progress.md | 6 +- docs/guides/storage-adapters.md | 4 +- docs/guides/subtypes-and-facets.md | 4 +- docs/guides/upgrading-7-to-8.md | 6 +- docs/guides/vue-integration.md | 6 +- docs/neural-extraction.md | 16 ++-- docs/transactions.md | 6 +- docs/universal-display-augmentation.md | 2 +- docs/vfs/PROJECTION_STRATEGY_API.md | 12 +-- docs/vfs/QUICK_START.md | 10 +- docs/vfs/README.md | 4 +- docs/vfs/ROADMAP.md | 6 +- docs/vfs/SEMANTIC_VFS.md | 2 +- docs/vfs/VFS_API_GUIDE.md | 4 +- docs/vfs/VFS_CORE.md | 4 +- docs/vfs/VFS_GRAPH_TYPES.md | 2 +- docs/vfs/VFS_INITIALIZATION.md | 6 +- docs/vfs/building-file-explorers.md | 6 +- examples/bluesky-distributed-setup.js | 2 +- examples/monitor-cache-performance.ts | 2 +- integrations/README.md | 8 +- integrations/google-sheets/README.md | 4 +- package-lock.json | 4 +- package.json | 11 ++- scripts/release.sh | 93 ++++--------------- src/brainy.ts | 6 +- src/db/errors.ts | 2 +- src/embeddings/wasm/modelLoader.ts | 8 +- src/errors/notFound.ts | 2 +- src/integrations/index.ts | 2 +- src/mcp/README.md | 4 +- src/plugin.ts | 4 +- src/types/brainy.types.ts | 2 +- src/types/reservedFields.ts | 2 +- src/utils/brainyTypes.ts | 2 +- src/utils/version.ts | 2 +- tests/unit/brainy/migration-deference.test.ts | 4 +- 74 files changed, 230 insertions(+), 284 deletions(-) diff --git a/.claude/skills/architecture.md b/.claude/skills/architecture.md index de046b17..4de3c287 100644 --- a/.claude/skills/architecture.md +++ b/.claude/skills/architecture.md @@ -2,7 +2,7 @@ ## What Is Brainy -@soulcraft/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. +@soulcraftlabs/brainy (v7.17.0) is a Universal Knowledge Protocol -- a Triple Intelligence database combining vector search, graph traversal, and metadata filtering in a single library. Published to npm as a public MIT-licensed package. ## Core Architecture diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml index a08875ae..58cb1d30 100644 --- a/.forgejo/workflows/publish-source.yml +++ b/.forgejo/workflows/publish-source.yml @@ -32,7 +32,7 @@ jobs: run: | set -eo pipefail - SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" + SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" VERSION="$(node -p "require('./package.json').version")" # The dist-tag follows the version: a prerelease (any hyphen — # 10.4.0-rc.1) publishes under 'rc' and must NEVER move 'latest' — @@ -43,13 +43,13 @@ jobs: case "$VERSION" in *-*) NPM_TAG="rc" ;; esac - echo "Publishing @soulcraft/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." + echo "Publishing @soulcraftlabs/brainy@${VERSION} to The Source registry (dist-tag: ${NPM_TAG})..." TMPRC="$(mktemp)" chmod 600 "$TMPRC" { - echo "@soulcraft:registry=${SOURCE_NPM_REG}" - echo "//source.soulcraft.com/api/packages/soulcraft/npm/:_authToken=${FORGE_NPM_TOKEN}" + echo "@soulcraftlabs:registry=${SOURCE_NPM_REG}" + echo "//source.soulcraft.com/api/packages/soulcraftlabs/npm/:_authToken=${FORGE_NPM_TOKEN}" } > "$TMPRC" # The release script bumps package.json's version before it tags, so @@ -64,7 +64,7 @@ jobs: # exit code: a benign duplicate publish (a prior run, or a mirror, already # landed this exact version) reports failure even though the registry # already holds the right content. - LANDED_VERSION="$(npm view "@soulcraft/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" + LANDED_VERSION="$(npm view "@soulcraftlabs/brainy@${VERSION}" version --userconfig "$TMPRC" 2>/dev/null || echo "")" rm -f "$TMPRC" if [ "$LANDED_VERSION" != "$VERSION" ]; then @@ -73,7 +73,7 @@ jobs: fi if [ "$PUBLISH_OK" = true ]; then - echo "Published and verified @soulcraft/brainy@${VERSION} on The Source registry." + echo "Published and verified @soulcraftlabs/brainy@${VERSION} on The Source registry." else - echo "::warning::npm publish reported failure, but readback confirms @soulcraft/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." + echo "::warning::npm publish reported failure, but readback confirms @soulcraftlabs/brainy@${VERSION} is already live on The Source (a prior run or mirror landed it) — treating this run as successful, since the registry content is correct. Any OTHER failure mode would have failed the readback check above instead." fi diff --git a/CLAUDE.md b/CLAUDE.md index 6acd0e0d..56df0b72 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,13 +12,13 @@ Handoff file: `/home/dpsifr/.strategy/PLATFORM-HANDOFF.md` **Brainy's current open actions:** None. MIT open-source — no platform-specific actions. -**Current version:** run `npm view @soulcraft/brainy version` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` +**Current version:** run `npm view @soulcraftlabs/brainy version --registry https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` (never trust a hardcoded number here — this line went stale for months); consumer-facing changes tracked in `RELEASES.md` --- ## Project Overview -Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraft/brainy` on npm under the MIT license. +Brainy is a Universal Knowledge Protocol -- a Triple Intelligence database that combines vector similarity search, graph traversal, and metadata filtering into a single TypeScript library. Published as `@soulcraftlabs/brainy` on The Source (source.soulcraft.com registry) under the MIT license. ## Getting Started diff --git a/README.md b/README.md index ca558340..47a9123a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- Brainy + Brainy

Brainy

@@ -11,8 +11,8 @@

- npm version - npm downloads + Package on The Source + Repository CI Documentation MIT License @@ -30,6 +30,8 @@ --- +**Open Brainy** is the MIT engine — the open API, client library, types, and protocol; an openly specified canonical on-disk format; and this TypeScript reference engine, scoped as a single-node engine for stores up to roughly one million rows. `@soulcraft/brainy` 10.4.2 was the last release under the old package name — the name passes to the native engine, **Brainy**, at 11.0.0: the same API over the same open format at production scale, and it requires a license. + Built because we were tired of stitching a vector store to a graph database to a document store — and spending weeks on plumbing before writing a line of business logic. Brainy indexes every fact **three ways at once** and lets one call query them together: | You write | Brainy indexes it as | You query it with | @@ -45,12 +47,14 @@ It runs **inside your process** — no server, no Docker, nothing to operate — ## Quick start ```bash -bun add @soulcraft/brainy # Bun ≥ 1.1 — recommended -npm install @soulcraft/brainy # Node.js ≥ 22 +bun add @soulcraftlabs/brainy # Bun ≥ 1.1 — recommended +npm install @soulcraftlabs/brainy # Node.js ≥ 22 ``` +> **Registry**: add `@soulcraftlabs:registry=https://source.soulcraft.com/api/packages/soulcraftlabs/npm/` to your `.npmrc` (anonymous read). + ```javascript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() // in-memory; one line swaps to disk await brain.init() diff --git a/SECURITY.md b/SECURITY.md index 1f3c4732..91d40d49 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -30,7 +30,7 @@ commit to backporting fixes to unsupported lines. ## Scope -This policy covers the `@soulcraft/brainy` package itself — the code in +This policy covers the `@soulcraftlabs/brainy` package itself — the code in this repository. If you're evaluating a deployment that also uses `@soulcraft/cor`, report issues in that package the same way, to the same address; we'll route internally. diff --git a/bin/brainy-ts.js b/bin/brainy-ts.js index 4e9aedb8..90a35e98 100644 --- a/bin/brainy-ts.js +++ b/bin/brainy-ts.js @@ -3,7 +3,7 @@ /** * Modern TypeScript CLI Runner * - * This is the entry point after npm install @soulcraft/brainy + * This is the entry point after npm install @soulcraftlabs/brainy * It runs the compiled TypeScript CLI code */ diff --git a/bun.lock b/bun.lock index c31b3865..1e3e66e2 100644 --- a/bun.lock +++ b/bun.lock @@ -3,7 +3,7 @@ "configVersion": 0, "workspaces": { "": { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", "@azure/identity": "^4.0.0", diff --git a/docs/DEVELOPER_LEARNING_PATH.md b/docs/DEVELOPER_LEARNING_PATH.md index 4134ae63..b2d22fb1 100644 --- a/docs/DEVELOPER_LEARNING_PATH.md +++ b/docs/DEVELOPER_LEARNING_PATH.md @@ -25,13 +25,13 @@ ### Prerequisites ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Your First Neural Database ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // Step 1: Create and initialize Brainy const brain = new Brainy({ @@ -143,7 +143,7 @@ Once you're comfortable with basic operations, move to **Level 2** to learn abou ### Building a Knowledge Graph ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -314,7 +314,7 @@ Ready for AI-powered search and clustering? Move to **Level 3**. ### Triple Intelligence in Action ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -529,7 +529,7 @@ Want to treat files as intelligent entities? Learn the **Virtual Filesystem** in ### Files as Intelligent Entities ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) await brain.init() @@ -832,7 +832,7 @@ Ready for production deployment? Level 5 covers **planet-scale architecture**. ### Production-Ready Deployment ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // 1. PRODUCTION STORAGE - Filesystem with off-site snapshots console.log('Initializing production storage...\n') diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 1cc38ce9..77fbbd79 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -1217,7 +1217,7 @@ where: { await brain.find({ type: 'Document' }) // ✅ Correct: Use NounType enum -import { NounType } from '@soulcraft/brainy' +import { NounType } from '@soulcraftlabs/brainy' await brain.find({ type: NounType.Document }) // ❌ Error: Operator not recognized diff --git a/docs/MIGRATION-V3-TO-V4.md b/docs/MIGRATION-V3-TO-V4.md index 29c409ac..680b6928 100644 --- a/docs/MIGRATION-V3-TO-V4.md +++ b/docs/MIGRATION-V3-TO-V4.md @@ -153,13 +153,13 @@ brainy-data/ ### Step 1: Update Brainy Package ```bash -npm install @soulcraft/brainy@latest +npm install @soulcraftlabs/brainy@latest ``` **Check your version:** ```bash -npm list @soulcraft/brainy -# Should show: @soulcraft/brainy@4.0.0 +npm list @soulcraftlabs/brainy +# Should show: @soulcraftlabs/brainy@4.0.0 ``` ### Step 2: No Code Changes Required! ✅ @@ -374,7 +374,7 @@ If you encounter issues, you can rollback: ```bash # Reinstall v3 -npm install @soulcraft/brainy@^3.50.0 +npm install @soulcraftlabs/brainy@^3.50.0 # Restart application ``` @@ -389,7 +389,7 @@ rm -rf ./data cp -r ./data-backup ./data # Reinstall v3 -npm install @soulcraft/brainy@^3.50.0 +npm install @soulcraftlabs/brainy@^3.50.0 ``` ## Common Migration Scenarios @@ -539,7 +539,7 @@ console.log('Storage type:', status.type) **Migration Checklist:** - ✅ Backup data -- ✅ Update npm package (`npm install @soulcraft/brainy@latest`) +- ✅ Update npm package (`npm install @soulcraftlabs/brainy@latest`) - ✅ Restart application (automatic migration) - ✅ Verify data integrity - ✅ Enable lifecycle policies diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index 3078c239..238d2252 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -46,7 +46,7 @@ If no plugin provides a given key, brainy uses its built-in JavaScript implement ### 1. Implement the `BrainyPlugin` interface ```typescript -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' const myPlugin: BrainyPlugin = { name: 'my-brainy-plugin', // Must be unique (typically your npm package name) @@ -90,7 +90,7 @@ await brain.init() **Programmatic registration:** For plugins not installed as npm packages, use `brain.use()`: ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import myPlugin from './my-plugin.js' const brain = new Brainy() @@ -272,10 +272,10 @@ When provided by an optional native acceleration plugin (such as `@soulcraft/cor #### `cache` **Type:** `UnifiedCache` -Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraft/brainy/internals`). +Replaces the global `UnifiedCache` singleton used for VFS path resolution, semantic caching, and vector index caching. Must implement the `UnifiedCache` interface (available from `@soulcraftlabs/brainy/internals`). ```typescript -import type { UnifiedCache } from '@soulcraft/brainy/internals' +import type { UnifiedCache } from '@soulcraftlabs/brainy/internals' context.registerProvider('cache', myNativeCache) ``` @@ -325,8 +325,8 @@ Plugins can register custom storage backends that users reference by name. ### Implementing a Storage Adapter ```typescript -import type { StorageAdapterFactory } from '@soulcraft/brainy/plugin' -import type { StorageAdapter } from '@soulcraft/brainy' +import type { StorageAdapterFactory } from '@soulcraftlabs/brainy/plugin' +import type { StorageAdapter } from '@soulcraftlabs/brainy' class MyStorageAdapter implements StorageAdapter { async init(): Promise { /* ... */ } @@ -360,9 +360,9 @@ Brainy provides three entry points for plugin developers: | Import Path | Contents | Stability | |-------------|----------|-----------| -| `@soulcraft/brainy` | Public API, types, StorageAdapter | Stable (semver) | -| `@soulcraft/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | -| `@soulcraft/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | +| `@soulcraftlabs/brainy` | Public API, types, StorageAdapter | Stable (semver) | +| `@soulcraftlabs/brainy/plugin` | BrainyPlugin, BrainyPluginContext, StorageAdapterFactory | Stable (semver) | +| `@soulcraftlabs/brainy/internals` | UnifiedCache, EntityIdMapper, logger utilities | Internal (may change between minor versions) | ## Diagnostics @@ -440,7 +440,7 @@ A minimal but useful plugin that provides SIMD-accelerated distance calculations ```typescript // simd-distance-plugin/src/plugin.ts -import type { BrainyPlugin, BrainyPluginContext } from '@soulcraft/brainy/plugin' +import type { BrainyPlugin, BrainyPluginContext } from '@soulcraftlabs/brainy/plugin' // Hypothetical native module import { simdCosineDistance } from './native.js' @@ -470,7 +470,7 @@ export default simdDistancePlugin "main": "./dist/plugin.js", "types": "./dist/plugin.d.ts", "peerDependencies": { - "@soulcraft/brainy": ">=7.0.0" + "@soulcraftlabs/brainy": ">=7.0.0" } } ``` @@ -478,7 +478,7 @@ export default simdDistancePlugin Usage: ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ plugins: ['brainy-simd-distance'] }) await brain.init() diff --git a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md index 4568cd31..ad4a4a40 100644 --- a/docs/PRODUCTION_SERVICE_ARCHITECTURE.md +++ b/docs/PRODUCTION_SERVICE_ARCHITECTURE.md @@ -54,7 +54,7 @@ After 40 API calls: ```typescript // server.ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // SINGLETON INSTANCE let brainInstance: Brainy | null = null @@ -174,7 +174,7 @@ process.on('SIGTERM', async () => { ```typescript // server.ts - Clean Bun implementation -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain: Brainy | null = null diff --git a/docs/README.md b/docs/README.md index 3290001f..ddb37d20 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,7 +5,7 @@ ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/RELEASE-GUIDE.md b/docs/RELEASE-GUIDE.md index 94c6a6cb..4b120bd8 100644 --- a/docs/RELEASE-GUIDE.md +++ b/docs/RELEASE-GUIDE.md @@ -99,7 +99,7 @@ Examples: ```bash # 1. Deprecate wrong version on npm -npm deprecate @soulcraft/brainy@X.X.X "Incorrect version - use Y.Y.Y" +npm deprecate @soulcraftlabs/brainy@X.X.X "Incorrect version - use Y.Y.Y" # 2. Fix version in package.json # 3. Republish correct version diff --git a/docs/SCALING.md b/docs/SCALING.md index e9ae1136..054d2096 100644 --- a/docs/SCALING.md +++ b/docs/SCALING.md @@ -13,7 +13,7 @@ ### In-Memory ```typescript -import Brainy from '@soulcraft/brainy' +import Brainy from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'memory' } }) ``` @@ -43,7 +43,7 @@ The native vector provider (via the optional `@soulcraft/cor` package) extends t Numbers below are **measured** by `tests/benchmarks/find-composition-scale.js` (a single Node 22 process, in-memory storage, 384-dim vectors, `balanced` recall). They are the -open-core (pure-TypeScript) path — what you get from `@soulcraft/brainy` with no native +open-core (pure-TypeScript) path — what you get from `@soulcraftlabs/brainy` with no native provider installed. Run it yourself: `node --max-old-space-size=8192 tests/benchmarks/find-composition-scale.js 100000`. `find()` query latency, p50 / p95 (200 queries each): diff --git a/docs/api/README.md b/docs/api/README.md index 82f48c91..ba49ff48 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -24,7 +24,7 @@ next: ## Quick Start ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() // Zero config! await brain.init() // VFS auto-initialized! @@ -1010,7 +1010,7 @@ await db.release() // unpin + free cached materialization ### Db API errors -All exported from `@soulcraft/brainy`: +All exported from `@soulcraftlabs/brainy`: | Error | Thrown by | Meaning | |---|---|---| @@ -1918,11 +1918,11 @@ isn't serving throws instead of rebuilding mid-query: | `MetadataIndexNotReadyError` | `find({ where })` | Metadata/field index isn't serving | | `VectorIndexNotReadyError` | `find({ query })`, `similar()` | Vector index isn't serving | -All three are exported from `@soulcraft/brainy`. Catch them to distinguish +All three are exported from `@soulcraftlabs/brainy`. Catch them to distinguish "index not ready" from a genuine empty result: ```typescript -import { MetadataIndexNotReadyError } from '@soulcraft/brainy' +import { MetadataIndexNotReadyError } from '@soulcraftlabs/brainy' try { const rows = await brain.find({ where: { status: 'active' } }) @@ -2208,7 +2208,7 @@ For the full taxonomy with all 169 types and their descriptions, see: - **📖 Documentation:** [Full Documentation](../) - **🐛 Issues:** [GitHub Issues](https://github.com/soulcraftlabs/brainy/issues) - **💬 Discussions:** [GitHub Discussions](https://github.com/soulcraftlabs/brainy/discussions) -- **📦 NPM:** [@soulcraft/brainy](https://www.npmjs.com/package/@soulcraft/brainy) +- **📦 NPM:** [@soulcraftlabs/brainy](https://www.npmjs.com/package/@soulcraftlabs/brainy) - **⭐ GitHub:** [Star us](https://github.com/soulcraftlabs/brainy) --- diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 48064757..83b9e23a 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -268,7 +268,7 @@ locks/_flush_responses/ # writer answers with .ack | **Counts/statistics** | Per-type and per-subtype maps | `_system/{type,subtype,verb-subtype}-statistics.json.gz`, `counts.json` | Recomputable by scanning entities (`brainy inspect repair`) | A pluggable index provider (the 8.0 plugin contract in -`@soulcraft/brainy/plugin`) may replace any of the JS implementations; the +`@soulcraftlabs/brainy/plugin`) may replace any of the JS implementations; the persisted formats above are contract-bound so JS and native implementations can interleave on the same directory. diff --git a/docs/architecture/finite-type-system.md b/docs/architecture/finite-type-system.md index 76492ee5..48a8b1fe 100644 --- a/docs/architecture/finite-type-system.md +++ b/docs/architecture/finite-type-system.md @@ -126,7 +126,7 @@ class TypeAwareMetadataIndex { **The Design**: Specify types clearly in your API calls: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' // Add entity with explicit type await brain.add({ @@ -231,7 +231,7 @@ class OrgEnrichmentAugmentation { **Brainy's Approach**: Extract **typed** concepts: ```typescript -import { NaturalLanguageProcessor } from '@soulcraft/brainy' +import { NaturalLanguageProcessor } from '@soulcraftlabs/brainy' const nlp = new NaturalLanguageProcessor() const concepts = await nlp.extractConcepts("Alice works at Google in San Francisco") @@ -382,7 +382,7 @@ import { getVerbTypes, BrainyTypes, suggestType -} from '@soulcraft/brainy' +} from '@soulcraftlabs/brainy' // Get all available noun types const nounTypes = getNounTypes() diff --git a/docs/architecture/multiprocess-storage-mixin.md b/docs/architecture/multiprocess-storage-mixin.md index 46f98398..1593bf8f 100644 --- a/docs/architecture/multiprocess-storage-mixin.md +++ b/docs/architecture/multiprocess-storage-mixin.md @@ -127,7 +127,7 @@ For reference, a clean migration path: `isMultiProcessSafe` type-guard. Keep `hasStorageMethod` for build/install artifact protection. 5. Document the new contract in `concepts/storage-adapters.md`. -6. Major-version-bump the `@soulcraft/brainy` peerDep range expected by +6. Major-version-bump the `@soulcraftlabs/brainy` peerDep range expected by plugins. Estimated work: ~half a day of code, ~2 hours of doc/example updates, diff --git a/docs/architecture/noun-verb-taxonomy.md b/docs/architecture/noun-verb-taxonomy.md index 286464be..3dac6892 100644 --- a/docs/architecture/noun-verb-taxonomy.md +++ b/docs/architecture/noun-verb-taxonomy.md @@ -20,7 +20,7 @@ next: Every example on this page is written against the real Brainy 8.0 API. The setup is always the same: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -40,7 +40,7 @@ Brainy's **Noun-Verb Taxonomy** achieves broad coverage of human knowledge throu - **Multi-hop Graph Traversals = Relationship Complexity** - **Result: Model data across virtually any industry** -Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraft/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. +Every piece of information can be represented as entities (nouns) connected by relationships (verbs) carrying properties (metadata). The standardized type system from `@soulcraftlabs/brainy` (`NounType`, `VerbType`) gives those nouns and verbs a stable, shared name. ## The Power of Standardization: Universal Interoperability diff --git a/docs/architecture/zero-config.md b/docs/architecture/zero-config.md index d42d6784..a35e6416 100644 --- a/docs/architecture/zero-config.md +++ b/docs/architecture/zero-config.md @@ -35,7 +35,7 @@ constructor and `init()`. ## Instant Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // That's it. No config needed. const brain = new Brainy() diff --git a/docs/concepts/field-addressing.md b/docs/concepts/field-addressing.md index c459021b..d24dd66b 100644 --- a/docs/concepts/field-addressing.md +++ b/docs/concepts/field-addressing.md @@ -167,7 +167,7 @@ await brain.find({ orderBy: 'createdAt' }) `UnresolvableFieldError` is exported from the package root: ```typescript -import { UnresolvableFieldError } from '@soulcraft/brainy' +import { UnresolvableFieldError } from '@soulcraftlabs/brainy' try { await brain.find({ orderBy: 'createdAt' }) diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md index 96199f11..923267df 100644 --- a/docs/concepts/index-health.md +++ b/docs/concepts/index-health.md @@ -95,7 +95,7 @@ catchable error naming the reason: | `MetadataIndexNotReadyError` | `find({ where })` | The metadata/field index isn't serving — a filtered read would otherwise return `[]` indistinguishable from "no matches" | | `VectorIndexNotReadyError` | `find({ query })`, `similar()` | The vector index isn't serving — a semantic search would otherwise return `[]` indistinguishable from "nothing similar" | -All three are exported from `@soulcraft/brainy`. Catch them where your application +All three are exported from `@soulcraftlabs/brainy`. Catch them where your application needs to distinguish "this index isn't ready yet" from "there's genuinely nothing here" — a health dashboard, a retry policy, an operator alert. The fix is always the same: reconcile the index, either by reopening the brain (which brings every diff --git a/docs/concepts/storage-adapters.md b/docs/concepts/storage-adapters.md index af6d068f..82aa01e8 100644 --- a/docs/concepts/storage-adapters.md +++ b/docs/concepts/storage-adapters.md @@ -61,7 +61,7 @@ The only required override is the capability flag. Returning `true` from to call `acquireWriterLock()` at init. ```typescript -import { FileSystemStorage } from '@soulcraft/brainy' +import { FileSystemStorage } from '@soulcraftlabs/brainy' export class MmapFileSystemStorage extends FileSystemStorage { public supportsMultiProcessLocking(): boolean { @@ -79,7 +79,7 @@ If your storage is **not filesystem-backed** (a custom network backend), extend `BaseStorage` directly: ```typescript -import { BaseStorage } from '@soulcraft/brainy' +import { BaseStorage } from '@soulcraftlabs/brainy' export class MyCloudStorage extends BaseStorage { // BaseStorage's default no-op implementations of the multi-process @@ -101,7 +101,7 @@ The defensive check at every new-storage-method call site (`brainy.ts`, `hasStorageMethod(name)`) does **not** exist to handle "plugin bundles a stale BaseStorage." Plugins ship a dist that preserves the dynamic ESM import (verify in your plugin's `dist/`: `import { FileSystemStorage } from -'@soulcraft/brainy'` is not rewritten to a vendored copy). The prototype +'@soulcraftlabs/brainy'` is not rewritten to a vendored copy). The prototype chain at runtime resolves to whatever Brainy version your consumer has installed. @@ -109,8 +109,8 @@ installed. the prototype chain at the consumer-app level: - **Stale `node_modules`** — a lingering install from before the consumer - upgraded Brainy. The package.json says `@soulcraft/brainy@7.22.0` but - `node_modules/@soulcraft/brainy` is still 7.20.x. + upgraded Brainy. The package.json says `@soulcraftlabs/brainy@7.22.0` but + `node_modules/@soulcraftlabs/brainy` is still 7.20.x. - **Lockfile drift** — `bun.lockb` / `package-lock.json` pins a brainy version older than the package.json range, and `bun install` honors the lockfile. @@ -131,7 +131,7 @@ and the warning names the adapter class plus a remediation hint: methods on its prototype chain. Writer locking and the flush-request RPC are disabled for this directory. Likely fix: clean install (`rm -rf node_modules bun.lockb && bun install`) or rebuild your container image to refresh -`@soulcraft/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. +`@soulcraftlabs/brainy` to ≥7.21. See docs/concepts/storage-adapters.md. ``` ## Authoring a new storage adapter — minimum checklist @@ -168,7 +168,7 @@ bun.lockb && bun install`) or rebuild your container image to refresh install time — fix install, not your plugin. 6. **Pin your peer dep generously.** `"peerDependencies": { - "@soulcraft/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin + "@soulcraftlabs/brainy": "^7.21.0" }` accepts any compatible 7.x. Don't pin to an exact patch unless you're tracking a known regression. ## Future direction @@ -185,5 +185,5 @@ follow-up; consumers don't need to anticipate the change. heartbeat semantics, what the lock protects. - [`guides/inspection`](../guides/inspection.md) — `brainy inspect` and the read-only mode. -- `node_modules/@soulcraft/brainy/dist/storage/baseStorage.d.ts` — the +- `node_modules/@soulcraftlabs/brainy/dist/storage/baseStorage.d.ts` — the authoritative type signatures for every method this page references. diff --git a/docs/guides/aggregation.md b/docs/guides/aggregation.md index 11d86ec8..616c8fc4 100644 --- a/docs/guides/aggregation.md +++ b/docs/guides/aggregation.md @@ -22,7 +22,7 @@ they share a single scan. ## Quick Start ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/framework-integration.md b/docs/guides/framework-integration.md index 984466c5..8f85da00 100644 --- a/docs/guides/framework-integration.md +++ b/docs/guides/framework-integration.md @@ -8,7 +8,7 @@ Brainy is **framework-friendly** - designed to drop into the server side of any Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed persistence layer. These belong on the server: -- **Zero configuration**: Just `import { Brainy } from '@soulcraft/brainy'` +- **Zero configuration**: Just `import { Brainy } from '@soulcraftlabs/brainy'` - **Auto storage detection**: `new Brainy()` auto-selects filesystem persistence on Node - **Cleaner code**: No browser polyfills, no conditional client/server imports - **Better DX**: One instance shared across your server routes @@ -18,13 +18,13 @@ Brainy embeds an HNSW vector index, a graph engine, and a filesystem-backed pers ### Install Brainy ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Integration ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Run on the server (API route, server component, backend service) // new Brainy() auto-detects filesystem persistence on Node @@ -105,7 +105,7 @@ On the server, create one Brainy instance and reuse it across requests. This mod ```javascript // lib/brain.server.js -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -163,7 +163,7 @@ On the server, create one Brainy instance and reuse it across requests: ```javascript // server/brain.js (server-only module) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -248,7 +248,7 @@ The matching backend endpoint uses Brainy directly (Node/Bun): ```typescript // server: api/search -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() // auto-detects filesystem persistence on Node await brain.init() @@ -266,7 +266,7 @@ In Next.js, Brainy lives in server code only: API routes, server components, or ```javascript // lib/brain.server.js (imported only by server code) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -318,7 +318,7 @@ Brainy runs in a server-only module (`*.server.js`); the component fetches resul ```javascript // src/lib/server/brain.js (server-only — note the .server suffix) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -432,7 +432,7 @@ import { defineConfig } from 'vite' export default defineConfig({ ssr: { - external: ['@soulcraft/brainy'] + external: ['@soulcraftlabs/brainy'] } }) ``` @@ -440,7 +440,7 @@ export default defineConfig({ ```javascript // rollup.config.js (server bundle) export default { - external: ['@soulcraft/brainy', 'node:fs', 'node:path', 'node:crypto'] + external: ['@soulcraftlabs/brainy', 'node:fs', 'node:path', 'node:crypto'] } ``` @@ -466,7 +466,7 @@ export async function load({ url }) { ```javascript // For build-time usage (runs in Node during the build) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' export async function generateStaticProps() { const brain = new Brainy({ @@ -513,7 +513,7 @@ export async function generateStaticProps() { ### Issue: Large client bundle size **Cause**: A client module is pulling in Brainy. -**Solution**: Move the `import { Brainy } from '@soulcraft/brainy'` into a server-only module so it never reaches the browser bundle. +**Solution**: Move the `import { Brainy } from '@soulcraftlabs/brainy'` into a server-only module so it never reaches the browser bundle. ### Issue: SSR hydration mismatch **Solution**: Run the search on the server (loader / server action / API route) and pass the results down as props, so server and client render the same markup. diff --git a/docs/guides/import-anything.md b/docs/guides/import-anything.md index b1bb15ef..ffabe55c 100644 --- a/docs/guides/import-anything.md +++ b/docs/guides/import-anything.md @@ -9,7 +9,7 @@ Brainy's import is **ONE magical method** that understands EVERYTHING: ## The Ultimate Simplicity ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/import-progress-examples.md b/docs/guides/import-progress-examples.md index 66f50713..18c3cb9a 100644 --- a/docs/guides/import-progress-examples.md +++ b/docs/guides/import-progress-examples.md @@ -13,7 +13,7 @@ Brainy provides real-time progress tracking for **all 7 supported file formats** ### Basic Progress Tracking ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import * as fs from 'fs' const brain = await Brainy.create() diff --git a/docs/guides/import-quick-reference.md b/docs/guides/import-quick-reference.md index 7837d49e..3bc26dae 100644 --- a/docs/guides/import-quick-reference.md +++ b/docs/guides/import-quick-reference.md @@ -7,7 +7,7 @@ ## Basic Import ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -187,7 +187,7 @@ await brain.import(file, { ## Complete Example ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import * as fs from 'fs' async function importCatalog() { diff --git a/docs/guides/inspection.md b/docs/guides/inspection.md index 240e81ae..8560b543 100644 --- a/docs/guides/inspection.md +++ b/docs/guides/inspection.md @@ -108,7 +108,7 @@ check fails — useful for piping into monitoring or CI. ## Programmatic inspection ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '/data/brain' } diff --git a/docs/guides/installation.md b/docs/guides/installation.md index 0a36f632..20d40ea2 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -21,21 +21,21 @@ next: ## Install ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` Or with your preferred package manager: ```bash -bun add @soulcraft/brainy -yarn add @soulcraft/brainy -pnpm add @soulcraft/brainy +bun add @soulcraftlabs/brainy +yarn add @soulcraftlabs/brainy +pnpm add @soulcraftlabs/brainy ``` ## Verify ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -52,7 +52,7 @@ npm install @soulcraft/cor ``` ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ plugins: ['@soulcraft/cor'] }) await brain.init() // native providers registered during init @@ -71,7 +71,7 @@ remains available on npm if you need it. Brainy ships with full TypeScript types. No `@types/` package needed: ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() diff --git a/docs/guides/migration-3.36.0.md b/docs/guides/migration-3.36.0.md index 8b1f239e..5f00534a 100644 --- a/docs/guides/migration-3.36.0.md +++ b/docs/guides/migration-3.36.0.md @@ -66,7 +66,7 @@ const results = await brain.search("query") **New diagnostics for capacity planning and performance tuning.** ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -112,7 +112,7 @@ Recommendations: ${stats.recommendations.join(', ')} ### Step 1: Update Package ```bash -npm install @soulcraft/brainy@latest +npm install @soulcraftlabs/brainy@latest ``` ### Step 2: Restart Your Application @@ -134,7 +134,7 @@ npm run start ### Check Adaptive Sizing is Working ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -218,7 +218,7 @@ For debugging or compatibility testing: If you need to rollback to v3.35.0: ```bash -npm install @soulcraft/brainy@3.35.0 +npm install @soulcraftlabs/brainy@3.35.0 ``` **Note:** We don't anticipate any issues, but rollback is straightforward if needed. @@ -367,7 +367,7 @@ if (stats.fairness.fairnessViolation) { ## Next Steps -1. ✅ **Upgrade:** `npm install @soulcraft/brainy@latest` +1. ✅ **Upgrade:** `npm install @soulcraftlabs/brainy@latest` 2. 📊 **Monitor:** Use `getCacheStats()` to verify performance improvements 3. 🎯 **Tune:** Adjust based on recommendations (if needed) 4. 📖 **Read:** [Operations Guide](../operations/capacity-planning.md) for capacity planning diff --git a/docs/guides/model-loading.md b/docs/guides/model-loading.md index e5b7b1d6..cc1b2b6a 100644 --- a/docs/guides/model-loading.md +++ b/docs/guides/model-loading.md @@ -37,7 +37,7 @@ This single WASM file contains everything needed for sentence embeddings. ```bash # Bun as a runtime — supported and recommended -bun add @soulcraft/brainy +bun add @soulcraftlabs/brainy bun run server.ts ``` diff --git a/docs/guides/namespace-migration.md b/docs/guides/namespace-migration.md index fad3c766..f7d2c7f7 100644 --- a/docs/guides/namespace-migration.md +++ b/docs/guides/namespace-migration.md @@ -80,7 +80,7 @@ If you read raw stored records (fact-log scanners, export tooling), use the exported shape-aware splitters — they handle both record eras: ```typescript -import { splitNounMetadataRecord } from '@soulcraft/brainy' +import { splitNounMetadataRecord } from '@soulcraftlabs/brainy' const { reserved, custom } = splitNounMetadataRecord(rawRecord) // reserved = engine fields · custom = the user's bag, ANY names ``` @@ -88,7 +88,7 @@ const { reserved, custom } = splitNounMetadataRecord(rawRecord) Feature detection (never version-sniff): ```typescript -import * as brainy from '@soulcraft/brainy' +import * as brainy from '@soulcraftlabs/brainy' const lawActive = 'FIELD_ADDRESSING_CAPABILITY' in brainy // 'field-addressing/v1' ``` diff --git a/docs/guides/nextjs-integration.md b/docs/guides/nextjs-integration.md index ab55e51f..25d6062d 100644 --- a/docs/guides/nextjs-integration.md +++ b/docs/guides/nextjs-integration.md @@ -9,7 +9,7 @@ Complete guide to integrating Brainy with Next.js applications, covering App Rou ```bash npx create-next-app@latest my-brainy-app cd my-brainy-app -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Setup @@ -18,7 +18,7 @@ npm install @soulcraft/brainy // app/components/BrainyProvider.jsx 'use client' import { createContext, useContext, useEffect, useState } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const BrainyContext = createContext() @@ -271,7 +271,7 @@ export default function SearchPage() { ```javascript // app/api/search/route.js (App Router) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -332,7 +332,7 @@ export async function GET() { ```javascript // pages/api/search.js (Pages Router) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -374,7 +374,7 @@ export default async function handler(req, res) { ```javascript // app/api/data/route.js -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -418,7 +418,7 @@ export async function POST(request) { ```jsx // app/actions/brainy.js 'use server' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brain = null @@ -630,7 +630,7 @@ CMD ["npm", "start"] /** @type {import('next').NextConfig} */ const nextConfig = { experimental: { - serverComponentsExternalPackages: ['@soulcraft/brainy'] + serverComponentsExternalPackages: ['@soulcraftlabs/brainy'] }, webpack: (config, { isServer }) => { if (!isServer) { @@ -797,7 +797,7 @@ export function rateLimit(req, limit = 100, window = 60000) { // app/contexts/BrainyContext.jsx 'use client' import { createContext, useContext, useReducer, useEffect } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const BrainyContext = createContext() @@ -873,7 +873,7 @@ import { BrainyProvider } from '../app/components/BrainyProvider' import { Search } from '../app/components/Search' // Mock Brainy -jest.mock('@soulcraft/brainy', () => ({ +jest.mock('@soulcraftlabs/brainy', () => ({ Brainy: jest.fn().mockImplementation(() => ({ init: jest.fn().mockResolvedValue(undefined), find: jest.fn().mockResolvedValue([ diff --git a/docs/guides/optimistic-concurrency.md b/docs/guides/optimistic-concurrency.md index 268bc5fa..2984998b 100644 --- a/docs/guides/optimistic-concurrency.md +++ b/docs/guides/optimistic-concurrency.md @@ -32,7 +32,7 @@ Brainy 7.31.0 adds a per-entity revision counter so multiple writers can coordin Every distributed-job scheduler eventually wants this exact loop: ```ts -import { Brainy, RevisionConflictError } from '@soulcraft/brainy' +import { Brainy, RevisionConflictError } from '@soulcraftlabs/brainy' const LOCK_ID = '...uuid for this job slot...' @@ -137,7 +137,7 @@ await brain.addIfMissing({ // ← not a real API It's race-prone as a plain read-then-write: two concurrent imports both see "not found," both insert, you get duplicates. Without a unique-index primitive (which Brainy doesn't have today), close the race with whole-store CAS — read at a pinned generation, then commit only if nothing moved: ```ts -import { GenerationConflictError } from '@soulcraft/brainy' +import { GenerationConflictError } from '@soulcraftlabs/brainy' async function addIfMissingByEmail(email: string, data: string) { for (let attempt = 0; attempt < 5; attempt++) { diff --git a/docs/guides/quick-start.md b/docs/guides/quick-start.md index 097c55fe..d9a4e896 100644 --- a/docs/guides/quick-start.md +++ b/docs/guides/quick-start.md @@ -18,13 +18,13 @@ Get Brainy running in under a minute. ## 1. Install ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ## 2. Initialize ```typescript -import { Brainy, NounType, VerbType } from '@soulcraft/brainy' +import { Brainy, NounType, VerbType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -67,7 +67,7 @@ await brain.relate({ ## 5. Query with Triple Intelligence ```typescript -import type { Result } from '@soulcraft/brainy' +import type { Result } from '@soulcraftlabs/brainy' // All three search paradigms in one call const results: Result[] = await brain.find({ diff --git a/docs/guides/standard-import-progress.md b/docs/guides/standard-import-progress.md index 9f2e2e5b..27dabe75 100644 --- a/docs/guides/standard-import-progress.md +++ b/docs/guides/standard-import-progress.md @@ -11,7 +11,7 @@ ### One Interface for Everything ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = await Brainy.create() @@ -78,7 +78,7 @@ interface ImportProgress { ```typescript import { useState } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' function UniversalImportProgress({ file }: { file: File }) { const [progress, setProgress] = useState({ @@ -177,7 +177,7 @@ function UniversalImportProgress({ file }: { file: File }) { ```typescript import ora from 'ora' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function importWithProgress(filePath: string) { const spinner = ora('Starting import...').start() diff --git a/docs/guides/storage-adapters.md b/docs/guides/storage-adapters.md index 06ec9f3a..a4224bc8 100644 --- a/docs/guides/storage-adapters.md +++ b/docs/guides/storage-adapters.md @@ -28,7 +28,7 @@ on-disk layout (memory's "disk" is a JS Map). ## Quick start ```ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Filesystem (recommended for any persistent workload): const brain = new Brainy({ @@ -134,7 +134,7 @@ config; the `type` is optional. If you want to skip the factory: ```ts -import { FileSystemStorage, MemoryStorage } from '@soulcraft/brainy' +import { FileSystemStorage, MemoryStorage } from '@soulcraftlabs/brainy' const fsStorage = new FileSystemStorage('./brainy-data') const memStorage = new MemoryStorage() diff --git a/docs/guides/subtypes-and-facets.md b/docs/guides/subtypes-and-facets.md index ff5de320..74311528 100644 --- a/docs/guides/subtypes-and-facets.md +++ b/docs/guides/subtypes-and-facets.md @@ -34,7 +34,7 @@ Three layers solve this: ### Write ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -240,7 +240,7 @@ await brain.migrateField({ A realistic adoption sequence for a brain that started without these primitives: ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy({ storage: { type: 'filesystem', path: './brain-data' } }) await brain.init() diff --git a/docs/guides/upgrading-7-to-8.md b/docs/guides/upgrading-7-to-8.md index a3c64fb9..53aa2a5c 100644 --- a/docs/guides/upgrading-7-to-8.md +++ b/docs/guides/upgrading-7-to-8.md @@ -25,7 +25,7 @@ content — and how 8.0 recovers it for you. ## TL;DR -- **Just upgrade to `@soulcraft/brainy@8.0.12` (or later) and open the store.** +- **Just upgrade to `@soulcraftlabs/brainy@8.0.12` (or later) and open the store.** If a previous upgrade left VFS content stranded, 8.0.12 **heals it on open**, with no operator action. - Want to force or script it? Call **`await brain.vfs.adoptOrphanedBlobs()`**. @@ -90,7 +90,7 @@ So the operator action for a stranded store is simply: **upgrade to 8.0.12 and open it.** ```ts -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Opening the store is all that is required — recovery runs during init(). const brain = new Brainy({ storage: { type: 'filesystem', path: '/data/my-store' } }) @@ -182,5 +182,5 @@ and opening each store is sufficient. The recovery is copy-only, so no rollback of the recovery itself is ever needed. If you need to roll back the **whole** 7→8 upgrade, restore the directory from your pre-upgrade backup (retained automatically while recovery is incomplete, or -your own snapshot) and pin `@soulcraft/brainy@7.x`. 8.0 does not keep the old +your own snapshot) and pin `@soulcraftlabs/brainy@7.x`. 8.0 does not keep the old branch layout in place, so a directory-level restore is the rollback path. diff --git a/docs/guides/vue-integration.md b/docs/guides/vue-integration.md index 7f7c6a06..34d18ebf 100644 --- a/docs/guides/vue-integration.md +++ b/docs/guides/vue-integration.md @@ -12,7 +12,7 @@ Complete guide to integrating Brainy with Vue.js applications, covering Vue 3, N npm create vue@latest my-brainy-app cd my-brainy-app npm install -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ### Basic Setup @@ -574,7 +574,7 @@ Nuxt's server engine (Nitro) is the natural home for Brainy: it runs on Node/Bun ```javascript // server/utils/brain.js (server-only — Nitro never bundles this into the client) -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' let brainPromise @@ -1201,7 +1201,7 @@ import vue from '@vitejs/plugin-vue' export default defineConfig({ plugins: [vue()], ssr: { - external: ['@soulcraft/brainy'] + external: ['@soulcraftlabs/brainy'] } }) ``` diff --git a/docs/neural-extraction.md b/docs/neural-extraction.md index 989b1b60..cfb6d764 100644 --- a/docs/neural-extraction.md +++ b/docs/neural-extraction.md @@ -24,7 +24,7 @@ Brainy's neural extraction system uses a **4-signal ensemble architecture** to c ### Method 1: Brain Instance (Recommended) ```typescript -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -62,9 +62,9 @@ const people = await brain.extractEntities('...', { import { SmartExtractor, SmartRelationshipExtractor -} from '@soulcraft/brainy' +} from '@soulcraftlabs/brainy' // Or use subpath imports: -import { SmartExtractor } from '@soulcraft/brainy/neural/SmartExtractor' +import { SmartExtractor } from '@soulcraftlabs/brainy/neural/SmartExtractor' const brain = new Brainy() await brain.init() @@ -176,7 +176,7 @@ const withVectors = await brain.extractEntities(text, { **Direct entity type classifier.** Use when you have pre-detected candidates or need custom configuration. ```typescript -import { SmartExtractor, FormatContext } from '@soulcraft/brainy' +import { SmartExtractor, FormatContext } from '@soulcraftlabs/brainy' const extractor = new SmartExtractor(brain, { minConfidence: 0.7, // Threshold @@ -229,7 +229,7 @@ interface ExtractionResult { **Relationship type classifier.** Determines verb/relationship types between entities. ```typescript -import { SmartRelationshipExtractor } from '@soulcraft/brainy' +import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' const relExtractor = new SmartRelationshipExtractor(brain, { minConfidence: 0.6, @@ -286,7 +286,7 @@ const rel = await relExtractor.infer( **Full extraction orchestrator.** Handles candidate detection, classification, and deduplication. ```typescript -import { NeuralEntityExtractor } from '@soulcraft/brainy' +import { NeuralEntityExtractor } from '@soulcraftlabs/brainy' const extractor = new NeuralEntityExtractor(brain) @@ -607,7 +607,7 @@ const locations = entities.filter(e => e.type === NounType.Location) ### Example 2: Excel Data Classification ```typescript -import { SmartExtractor } from '@soulcraft/brainy' +import { SmartExtractor } from '@soulcraftlabs/brainy' const extractor = new SmartExtractor(brain) @@ -629,7 +629,7 @@ for (let i = 0; i < cells.length; i++) { ### Example 3: Relationship Extraction ```typescript -import { SmartRelationshipExtractor } from '@soulcraft/brainy' +import { SmartRelationshipExtractor } from '@soulcraftlabs/brainy' const relExtractor = new SmartRelationshipExtractor(brain) diff --git a/docs/transactions.md b/docs/transactions.md index fce7d10e..cbea39c0 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -204,8 +204,8 @@ await brain.add({ data: { name: 'Entity' }, type: NounType.Thing }) ### Basic Add Operation ```typescript -import { Brainy } from '@soulcraft/brainy' -import { NounType } from '@soulcraft/brainy/types' +import { Brainy } from '@soulcraftlabs/brainy' +import { NounType } from '@soulcraftlabs/brainy/types' const brain = new Brainy() await brain.init() @@ -428,7 +428,7 @@ await brain.relate({ ... }) // a crash here leaves the entity unlinked ```typescript import { describe, it, expect } from 'vitest' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' describe('Transaction Tests', () => { it('should rollback on failure', async () => { diff --git a/docs/universal-display-augmentation.md b/docs/universal-display-augmentation.md index da42874c..464b91fb 100644 --- a/docs/universal-display-augmentation.md +++ b/docs/universal-display-augmentation.md @@ -23,7 +23,7 @@ The Universal Display Augmentation is a powerful AI-powered system that automati ### Basic Usage ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brainy = new Brainy() await brainy.init() diff --git a/docs/vfs/PROJECTION_STRATEGY_API.md b/docs/vfs/PROJECTION_STRATEGY_API.md index 380862e1..f1319d5b 100644 --- a/docs/vfs/PROJECTION_STRATEGY_API.md +++ b/docs/vfs/PROJECTION_STRATEGY_API.md @@ -71,9 +71,9 @@ Let's build a projection that organizes files by priority (high, medium, low): ### Step 1: Create the Strategy Class ```typescript -import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' -import { Brainy } from '@soulcraft/brainy' -import { VirtualFileSystem, VFSEntity } from '@soulcraft/brainy/vfs' +import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' +import { Brainy } from '@soulcraftlabs/brainy' +import { VirtualFileSystem, VFSEntity } from '@soulcraftlabs/brainy/vfs' export class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' @@ -141,7 +141,7 @@ export class PriorityProjection extends BaseProjectionStrategy { ### Step 2: Register the Strategy ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { PriorityProjection } from './PriorityProjection' const brain = new Brainy() @@ -537,7 +537,7 @@ Use the projection's resolve cache: ```typescript import { describe, it, expect, beforeAll } from 'vitest' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { PriorityProjection } from './PriorityProjection' describe('PriorityProjection', () => { @@ -714,7 +714,7 @@ async resolve(brain, vfs, value: string) { 3. Use appropriate limits: Don't fetch more than needed ### Type errors -1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraft/brainy'` +1. Import correct types: `import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy'` 2. Use `as VFSEntity` when mapping results 3. Check BaseProjectionStrategy import diff --git a/docs/vfs/QUICK_START.md b/docs/vfs/QUICK_START.md index 8b0efce6..4a1f83dc 100644 --- a/docs/vfs/QUICK_START.md +++ b/docs/vfs/QUICK_START.md @@ -14,11 +14,11 @@ A file explorer that: ## ⚡ Step 1: Basic Setup (1 minute) ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // ✅ CORRECT: Use filesystem storage for production const brain = new Brainy({ @@ -115,7 +115,7 @@ Here's a complete React component using the correct patterns: ```tsx import React, { useState, useEffect } from 'react' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' export function FileExplorer() { const [brain, setBrain] = useState(null) @@ -288,8 +288,8 @@ Your file explorer is now working! Here's what to explore next: ### "Module not found" errors ```bash # Make sure you're using the right import -npm ls @soulcraft/brainy # Check version -npm install @soulcraft/brainy@latest # Update if needed +npm ls @soulcraftlabs/brainy # Check version +npm install @soulcraftlabs/brainy@latest # Update if needed ``` ### "VFS not initialized" errors diff --git a/docs/vfs/README.md b/docs/vfs/README.md index b95f0d7b..a94910c9 100644 --- a/docs/vfs/README.md +++ b/docs/vfs/README.md @@ -24,7 +24,7 @@ Brainy VFS is a revolutionary virtual filesystem that runs on top of Brainy's ne ## Quick Start ```javascript -import { VirtualFileSystem } from '@soulcraft/brainy/vfs' +import { VirtualFileSystem } from '@soulcraftlabs/brainy/vfs' // Initialize the VFS const vfs = new VirtualFileSystem({ @@ -381,7 +381,7 @@ Brainy VFS fully leverages Brainy's revolutionary Triple Intelligence system: ## Installation ```bash -npm install @soulcraft/brainy +npm install @soulcraftlabs/brainy ``` ## Requirements diff --git a/docs/vfs/ROADMAP.md b/docs/vfs/ROADMAP.md index 93c5b901..c8d15cd2 100644 --- a/docs/vfs/ROADMAP.md +++ b/docs/vfs/ROADMAP.md @@ -135,7 +135,7 @@ Mount VFS as a native filesystem on Linux/Mac/Windows. ```typescript // Planned (research phase) -import { mountVFS } from '@soulcraft/brainy/vfs/fuse' +import { mountVFS } from '@soulcraftlabs/brainy/vfs/fuse' await mountVFS(vfs, { mountPoint: '/mnt/brainy', @@ -160,7 +160,7 @@ These features would benefit from community contributions. If you're interested ### Express.js Static Middleware ```typescript // Wanted: Community contribution -import { createStaticMiddleware } from '@soulcraft/brainy/vfs/express' +import { createStaticMiddleware } from '@soulcraftlabs/brainy/vfs/express' app.use('/files', createStaticMiddleware(vfs, { index: ['index.html', 'index.md'], @@ -172,7 +172,7 @@ app.use('/files', createStaticMiddleware(vfs, { ### VSCode Extension ```typescript // Wanted: Community contribution -import { VFSProvider } from '@soulcraft/brainy/vfs/vscode' +import { VFSProvider } from '@soulcraftlabs/brainy/vfs/vscode' const provider = new VFSProvider(vfs) vscode.workspace.registerFileSystemProvider('brainy', provider) diff --git a/docs/vfs/SEMANTIC_VFS.md b/docs/vfs/SEMANTIC_VFS.md index 9298c822..f34ee9ae 100644 --- a/docs/vfs/SEMANTIC_VFS.md +++ b/docs/vfs/SEMANTIC_VFS.md @@ -327,7 +327,7 @@ console.log(id1 === id2 && id2 === id3) // true Create your own semantic dimensions: ```typescript -import { BaseProjectionStrategy } from '@soulcraft/brainy/vfs/semantic' +import { BaseProjectionStrategy } from '@soulcraftlabs/brainy/vfs/semantic' class PriorityProjection extends BaseProjectionStrategy { readonly name = 'priority' diff --git a/docs/vfs/VFS_API_GUIDE.md b/docs/vfs/VFS_API_GUIDE.md index e0c6a94c..5dcaaeb8 100644 --- a/docs/vfs/VFS_API_GUIDE.md +++ b/docs/vfs/VFS_API_GUIDE.md @@ -7,7 +7,7 @@ Brainy's Virtual Filesystem (VFS) provides a POSIX-like filesystem interface tha ## Quick Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Initialize Brainy const brain = new Brainy({ @@ -598,7 +598,7 @@ const user = await store.findById('users', 'user123') VFS uses standard POSIX-style errors: ```typescript -import { VFSError, VFSErrorCode } from '@soulcraft/brainy' +import { VFSError, VFSErrorCode } from '@soulcraftlabs/brainy' try { await vfs.readFile('/nonexistent.txt') diff --git a/docs/vfs/VFS_CORE.md b/docs/vfs/VFS_CORE.md index 1eeaf9f8..c1d502c0 100644 --- a/docs/vfs/VFS_CORE.md +++ b/docs/vfs/VFS_CORE.md @@ -280,7 +280,7 @@ GitBridge provides Git import/export capabilities: #### GitBridge Usage ```javascript // Import and instantiate GitBridge -import { GitBridge } from '@soulcraft/brainy' +import { GitBridge } from '@soulcraftlabs/brainy' const gitBridge = new GitBridge(vfs, brain) // Export VFS to Git repository structure @@ -452,7 +452,7 @@ This ordering prevents race conditions where file writes might fail because pare ## Complete Example ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function vfsExample() { // Initialize diff --git a/docs/vfs/VFS_GRAPH_TYPES.md b/docs/vfs/VFS_GRAPH_TYPES.md index 3c1f30f0..478bef7f 100644 --- a/docs/vfs/VFS_GRAPH_TYPES.md +++ b/docs/vfs/VFS_GRAPH_TYPES.md @@ -196,5 +196,5 @@ await brain.relate({ Always import and use the type enums: ```javascript -import { NounType, VerbType } from '@soulcraft/brainy' +import { NounType, VerbType } from '@soulcraftlabs/brainy' ``` \ No newline at end of file diff --git a/docs/vfs/VFS_INITIALIZATION.md b/docs/vfs/VFS_INITIALIZATION.md index 97e6b0bf..fd12fc71 100644 --- a/docs/vfs/VFS_INITIALIZATION.md +++ b/docs/vfs/VFS_INITIALIZATION.md @@ -5,7 +5,7 @@ The Brainy VFS is automatically initialized during `brain.init()`. No separate initialization needed! ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' // Create and initialize Brainy const brain = new Brainy({ @@ -71,7 +71,7 @@ VFS stores files as entities and relationships in the same graph as everything e ## Complete Example ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' async function useVFS() { // Initialize Brainy @@ -100,7 +100,7 @@ useVFS().catch(console.error) ## TypeScript Usage ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' class FileManager { private brain: Brainy diff --git a/docs/vfs/building-file-explorers.md b/docs/vfs/building-file-explorers.md index 6bb31871..7514c12e 100644 --- a/docs/vfs/building-file-explorers.md +++ b/docs/vfs/building-file-explorers.md @@ -37,7 +37,7 @@ Brainy VFS provides safe, tree-aware methods that prevent these issues: ### Method 1: Use `getDirectChildren()` (Recommended) ```typescript -import { Brainy, VirtualFileSystem } from '@soulcraft/brainy' +import { Brainy, VirtualFileSystem } from '@soulcraftlabs/brainy' const brain = new Brainy() await brain.init() @@ -97,7 +97,7 @@ Here's a complete example using React: ```tsx import React, { useState, useEffect } from 'react' -import { VirtualFileSystem } from '@soulcraft/brainy' +import { VirtualFileSystem } from '@soulcraftlabs/brainy' interface FileNode { name: string @@ -177,7 +177,7 @@ function TreeView({ node, onToggle, expanded }) { If you must build trees manually from flat lists, use the `VFSTreeUtils`: ```typescript -import { VFSTreeUtils } from '@soulcraft/brainy/vfs' +import { VFSTreeUtils } from '@soulcraftlabs/brainy/vfs' // Get all entities somehow const allEntities = await vfs.getDescendants('/root') diff --git a/examples/bluesky-distributed-setup.js b/examples/bluesky-distributed-setup.js index 9e83cf25..e3b33506 100644 --- a/examples/bluesky-distributed-setup.js +++ b/examples/bluesky-distributed-setup.js @@ -7,7 +7,7 @@ * the Bluesky firehose with Brainy's distributed architecture */ -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' import { WebSocket } from 'ws' // ===================================================== diff --git a/examples/monitor-cache-performance.ts b/examples/monitor-cache-performance.ts index 9d50d476..87c965a2 100644 --- a/examples/monitor-cache-performance.ts +++ b/examples/monitor-cache-performance.ts @@ -14,7 +14,7 @@ * ts-node examples/monitor-cache-performance.ts */ -import { Brainy, NounType } from '@soulcraft/brainy' +import { Brainy, NounType } from '@soulcraftlabs/brainy' // ANSI color codes for pretty output const colors = { diff --git a/integrations/README.md b/integrations/README.md index aa3d795b..de156623 100644 --- a/integrations/README.md +++ b/integrations/README.md @@ -5,7 +5,7 @@ Connect Brainy to spreadsheets, BI tools, and external systems with zero configu ## Quick Start ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -178,7 +178,7 @@ Webhooks include `X-Brainy-Signature` header with HMAC-SHA256 signature. ### Minimal (in-memory): ```typescript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -194,7 +194,7 @@ console.log(brain.hub.getInstructions()) ```typescript import express from 'express' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = express() const brain = new Brainy({ @@ -232,7 +232,7 @@ app.listen(3000, () => { ```typescript import { Hono } from 'hono' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = new Hono() diff --git a/integrations/google-sheets/README.md b/integrations/google-sheets/README.md index b2b0af3a..8309a30a 100644 --- a/integrations/google-sheets/README.md +++ b/integrations/google-sheets/README.md @@ -99,7 +99,7 @@ Add the `BRAINY_URL` script property in Apps Script settings. The simplest way to enable all integrations: ```javascript -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const brain = new Brainy({ integrations: true }) await brain.init() @@ -112,7 +112,7 @@ With Express: ```javascript import express from 'express' -import { Brainy } from '@soulcraft/brainy' +import { Brainy } from '@soulcraftlabs/brainy' const app = express() const brain = new Brainy({ integrations: true }) diff --git a/package-lock.json b/package-lock.json index fb88681f..9cc3c6b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "version": "10.4.2", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "version": "10.4.2", "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 17983c19..2312fcb0 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "@soulcraft/brainy", + "name": "@soulcraftlabs/brainy", "version": "10.4.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", @@ -126,15 +126,16 @@ "license": "MIT", "private": false, "publishConfig": { - "access": "public" + "access": "public", + "registry": "https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" }, - "homepage": "https://source.soulcraft.com/soulcraft/brainy", + "homepage": "https://source.soulcraft.com/soulcraftlabs/open-brainy", "bugs": { - "url": "https://source.soulcraft.com/soulcraft/brainy/issues" + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/issues" }, "repository": { "type": "git", - "url": "git+https://source.soulcraft.com/soulcraft/brainy.git" + "url": "git+https://source.soulcraft.com/soulcraftlabs/open-brainy.git" }, "files": [ "dist/**/*.js", diff --git a/scripts/release.sh b/scripts/release.sh index 5d03e66b..be1d6d6b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -15,11 +15,11 @@ NC='\033[0m' # No Color RELEASE_TYPE="${1:-patch}" # patch, minor, or major SKIP_TESTS=false DRY_RUN=false -# --source-only: the HOME leg only — tag, CI's publish to The Source, and the -# release page; NO storefront (npmjs) publish, NO pair verification, NO docs -# push. The pair-gate shape: a prerelease the fleet's other engine devDeps -# from our own registry while the pair is proven, never a public artifact. -# Refused for a non-prerelease version — a public floor is always a pair. +# --source-only is now a no-op: The Source is the one registry, so every +# release already ships Source-only — tag, CI's publish to The Source, the +# release page, and the docs push, with no separate storefront leg to skip. +# The flag is still accepted (for backward-compatible invocations) and just +# prints a notice; it no longer changes behavior. SOURCE_ONLY=false for arg in "$@"; do @@ -109,7 +109,7 @@ else ;; *) echo -e "${RED}❌ Invalid release type: ${RELEASE_TYPE}${NC}" - echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run] [--source-only (prereleases only)]" + echo "Usage: ./scripts/release.sh [patch|minor|major|] [--dry-run] [--source-only (no-op; The Source is the one registry)]" exit 1 ;; esac @@ -129,11 +129,7 @@ if [ "$PRERELEASE" = true ]; then echo -e "${YELLOW}⚠️ Prerelease → npm dist-tag '${NPM_TAG}', GitHub prerelease${NC}" fi if [ "$SOURCE_ONLY" = true ]; then - if [ "$PRERELEASE" != true ]; then - echo -e "${RED}❌ --source-only is for prereleases only: a non-prerelease version is a public floor and always ships as the byte-identical pair.${NC}" - exit 1 - fi - echo -e "${YELLOW}⚠️ --source-only → The Source (home) ONLY: no npmjs publish, no pair verification, no docs push${NC}" + echo -e "${YELLOW}⚠️ The Source is the one registry; --source-only is implied${NC}" fi echo "" @@ -209,9 +205,9 @@ echo -e "${GREEN}✅ Pushed to origin${NC}\n" # .forgejo/workflows/publish-source.yml, which builds and publishes on The # Source's own runner (datacenter-side: seconds, not the laptop's WAN timing # out on an 87MB tarball PUT). The laptop holds no home-registry publish -# credential anymore; it only waits for CI's result before trusting the -# home/npmjs pair enough to publish the storefront leg. -SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraft/npm/" +# credential anymore; it only waits for CI's result before continuing on to +# the release page and the docs push. +SOURCE_NPM_REG="https://source.soulcraft.com/api/packages/soulcraftlabs/npm/" SOURCE_POLL_INTERVAL_S=15 SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequential and a busy day's ci.yml # backlog has twice exceeded the old 20-minute window (8.10.3, 9.0.0); @@ -219,7 +215,7 @@ SOURCE_POLL_MAX_ATTEMPTS=200 # 200 × 15s = 50 minutes — the runner is sequen echo -e "${BLUE}9️⃣ Waiting for CI to publish v${NEW_VERSION} to The Source registry (home)...${NC}" SOURCE_LANDED=false for ((attempt = 1; attempt <= SOURCE_POLL_MAX_ATTEMPTS; attempt++)); do - LANDED_VERSION=$(npm view "@soulcraft/brainy@${NEW_VERSION}" version "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") + LANDED_VERSION=$(npm view "@soulcraftlabs/brainy@${NEW_VERSION}" version "--@soulcraftlabs:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "") if [ "$LANDED_VERSION" = "$NEW_VERSION" ]; then SOURCE_LANDED=true break @@ -232,57 +228,11 @@ if [ "$SOURCE_LANDED" = true ]; then echo -e "${GREEN}✅ CI published v${NEW_VERSION} to The Source${NC}\n" else echo -e "${RED}❌ CI's home publish did not land — check the workflow run on The Source; the pair must not diverge.${NC}" - echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraft/brainy@${NEW_VERSION} never became visible on the${NC}" - echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting before npmjs.${NC}" + echo -e "${RED} v${NEW_VERSION} was tagged and pushed, but @soulcraftlabs/brainy@${NEW_VERSION} never became visible on the${NC}" + echo -e "${RED} Source registry after ${SOURCE_POLL_MAX_ATTEMPTS} attempts, ${SOURCE_POLL_INTERVAL_S}s apart. Aborting.${NC}" exit 1 fi -if [ "$SOURCE_ONLY" = true ]; then - echo -e "${YELLOW}9️⃣½ Storefront (npmjs) leg SKIPPED — --source-only: v${NEW_VERSION} lives on The Source under dist-tag '${NPM_TAG}' only${NC}\n" -else - echo -e "${BLUE}9️⃣½ Publishing to npmjs (storefront, dist-tag: ${NPM_TAG})...${NC}" - # BYTE-IDENTITY LAW: the storefront republishes CI's EXACT artifact — download - # the tarball The Source serves and publish that file, never a fresh local pack - # (a local rebuild can differ byte-wise, and the fleet verifies the pair by - # shasum across registries). - STOREFRONT_TMP="$(mktemp -d)" - (cd "$STOREFRONT_TMP" && npm pack "@soulcraft/brainy@${NEW_VERSION}" "--@soulcraft:registry=${SOURCE_NPM_REG}" >/dev/null) - SOURCE_TARBALL="$(ls "$STOREFRONT_TMP"/soulcraft-brainy-*.tgz)" - echo -e "${BLUE} home artifact: $(sha256sum "$SOURCE_TARBALL" | cut -d' ' -f1)${NC}" - npm publish "$SOURCE_TARBALL" --tag "$NPM_TAG" "--@soulcraft:registry=https://registry.npmjs.org/" - rm -rf "$STOREFRONT_TMP" - # Brainy is the only PUBLIC @soulcraft package — verify visibility after every publish. - npm access get status @soulcraft/brainy "--@soulcraft:registry=https://registry.npmjs.org/" || true - # Verify the pair is byte-identical by registry-reported shasum — divergence - # here means the storefront leg must be treated as failed, loudly. RETRIED - # with raw curl: npmjs metadata propagates with a lag measured in minutes, - # and a one-shot npm-view probe fired a false DIVERGENCE on 10.0.0 while a - # raw curl of the registry document already confirmed byte-identity. The - # probe now reads the registry JSON directly (no npm cache in the path) and - # gives propagation up to 5 minutes before calling the pair divergent. - NPMJS_VERIFY_ATTEMPTS=20 - NPMJS_VERIFY_INTERVAL_S=15 # 20 × 15s = 5 minutes of propagation grace - SOURCE_SHA=$(npm view "@soulcraft/brainy@${NEW_VERSION}" dist.shasum "--@soulcraft:registry=${SOURCE_NPM_REG}" 2>/dev/null || echo "source-unavailable") - PAIR_IDENTICAL=false - for ((attempt = 1; attempt <= NPMJS_VERIFY_ATTEMPTS; attempt++)); do - NPMJS_SHA=$(curl -fsSL "https://registry.npmjs.org/@soulcraft%2Fbrainy" 2>/dev/null \ - | node -e "let d='';process.stdin.on('data',c=>d+=c).on('end',()=>{try{const v=JSON.parse(d).versions[process.argv[1]];console.log(v?v.dist.shasum:'')}catch{console.log('')}})" "${NEW_VERSION}" \ - || echo "") - if [ -n "$NPMJS_SHA" ] && [ "$SOURCE_SHA" = "$NPMJS_SHA" ]; then - PAIR_IDENTICAL=true - break - fi - echo -e "${YELLOW} … npmjs metadata not settled (attempt ${attempt}/${NPMJS_VERIFY_ATTEMPTS}: '${NPMJS_SHA:-absent}' vs '${SOURCE_SHA}'); retrying in ${NPMJS_VERIFY_INTERVAL_S}s${NC}" - sleep "$NPMJS_VERIFY_INTERVAL_S" - done - if [ "$PAIR_IDENTICAL" = true ]; then - echo -e "${GREEN}✅ Published to npmjs — byte-identical pair (shasum ${NPMJS_SHA})${NC}\n" - else - echo -e "${RED}❌ REGISTRY DIVERGENCE: The Source shasum ${SOURCE_SHA} != npmjs shasum ${NPMJS_SHA} after ${NPMJS_VERIFY_ATTEMPTS} attempts — investigate before announcing${NC}\n" - exit 1 - fi -fi - # Step 11: Release object on The Source (presentational — the tag, CHANGELOG, # and RELEASES.md are the record; this just gives The Source's UI a release page). echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" @@ -303,24 +253,15 @@ fi # DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — # that already happened) when a push errors, so the docs site never # silently trails npm. -if [ "$SOURCE_ONLY" = true ]; then - echo -e "${YELLOW}1️⃣2️⃣ Docs push SKIPPED — --source-only (a home-only prerelease publishes no public docs)${NC}\n" +echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" +if node scripts/push-docs.js; then + echo -e "${GREEN}✅ Docs push step done${NC}\n" else - echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" - if node scripts/push-docs.js; then - echo -e "${GREEN}✅ Docs push step done${NC}\n" - else - echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" - fi + echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" fi echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" -if [ "$SOURCE_ONLY" = true ]; then - echo -e "📦 npmjs: ${YELLOW}not published (--source-only)${NC}" -else - echo -e "📦 npm: ${BLUE}https://www.npmjs.com/package/@soulcraft/brainy/v/${NEW_VERSION}${NC}" -fi echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" diff --git a/src/brainy.ts b/src/brainy.ts index ed958a67..966d3188 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -952,14 +952,14 @@ export class Brainy implements BrainyInterface { * extends FileSystemStorage`) inherit new methods Brainy adds to * `FileSystemStorage` / `BaseStorage` automatically — `typeof` walks the * prototype chain, so there's no in-package version skew to worry about as - * long as the plugin's own dist resolves `@soulcraft/brainy` dynamically + * long as the plugin's own dist resolves `@soulcraftlabs/brainy` dynamically * (which Cortex 2.2.x onward does — see * `node_modules/@soulcraft/cor/dist/storage/mmapFileSystemStorage.js`). * * This helper exists for the **build/install** failure modes the import * resolution can't catch: * - Stale `node_modules` left over from a prior `bun install` against - * `@soulcraft/brainy ≤7.20.x`. + * `@soulcraftlabs/brainy ≤7.20.x`. * - Lockfile drift pinning brainy below the version that introduced the * method. * - Docker layer caches that reuse a `node_modules` from an earlier image. @@ -1175,7 +1175,7 @@ export class Brainy implements BrainyInterface { `and the flush-request RPC are disabled for this directory. ` + `Likely fix: clean install (\`rm -rf node_modules bun.lockb && ` + `bun install\`) or rebuild your container image to refresh ` + - `\`@soulcraft/brainy\` to ≥7.21. See docs/concepts/storage-adapters.md.` + `\`@soulcraftlabs/brainy\` to ≥7.21. See docs/concepts/storage-adapters.md.` ) } else { console.warn( diff --git a/src/db/errors.ts b/src/db/errors.ts index e20488f8..3b4c1af6 100644 --- a/src/db/errors.ts +++ b/src/db/errors.ts @@ -28,7 +28,7 @@ * speculative `with()` overlay; the canonical storage walk only ever answers * "what is live right now." * - * All are exported from the package root (`@soulcraft/brainy`). + * All are exported from the package root (`@soulcraftlabs/brainy`). */ /** diff --git a/src/embeddings/wasm/modelLoader.ts b/src/embeddings/wasm/modelLoader.ts index 45ffc4d3..b39d90ea 100644 --- a/src/embeddings/wasm/modelLoader.ts +++ b/src/embeddings/wasm/modelLoader.ts @@ -128,7 +128,7 @@ async function loadBunAssets(): Promise { } // Strategy 2: node_modules path relative to CWD (for installed packages) - const nmPath = './node_modules/@soulcraft/brainy/assets/models/all-MiniLM-L6-v2' + const nmPath = './node_modules/@soulcraftlabs/brainy/assets/models/all-MiniLM-L6-v2' pathsToTry.push([ `${nmPath}/model.safetensors`, `${nmPath}/tokenizer.json`, @@ -168,9 +168,9 @@ async function loadBunAssets(): Promise { // If all strategies fail, provide helpful error message throw new Error( 'Could not load model assets. For bun --compile, ensure model files are accessible:\n' + - ' Option 1: Keep node_modules/@soulcraft/brainy/assets/ alongside your binary\n' + + ' Option 1: Keep node_modules/@soulcraftlabs/brainy/assets/ alongside your binary\n' + ' Option 2: Copy assets/ folder to your working directory\n' + - ' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraft/brainy/assets/**/*"' + ' Option 3: Use --asset flag: bun build --compile --asset="./node_modules/@soulcraftlabs/brainy/assets/**/*"' ) } @@ -190,7 +190,7 @@ async function loadNodeAssets(): Promise { if (!fs.existsSync(assetsDir)) { throw new Error( `Model assets not found: ${assetsDir}\n` + - `Ensure @soulcraft/brainy is installed correctly.` + `Ensure @soulcraftlabs/brainy is installed correctly.` ) } diff --git a/src/errors/notFound.ts b/src/errors/notFound.ts index 8eca9b2e..628797a6 100644 --- a/src/errors/notFound.ts +++ b/src/errors/notFound.ts @@ -14,7 +14,7 @@ * - {@link RelationNotFoundError} — a referenced relationship (verb) does * not exist. * - * Both are exported from the package root (`@soulcraft/brainy`). + * Both are exported from the package root (`@soulcraftlabs/brainy`). */ /** diff --git a/src/integrations/index.ts b/src/integrations/index.ts index 757a9fe5..6a6734d9 100644 --- a/src/integrations/index.ts +++ b/src/integrations/index.ts @@ -9,7 +9,7 @@ * * @example Enable integrations (recommended) * ```typescript - * import { Brainy } from '@soulcraft/brainy' + * import { Brainy } from '@soulcraftlabs/brainy' * * const brain = new Brainy({ integrations: true }) * await brain.init() diff --git a/src/mcp/README.md b/src/mcp/README.md index c69a3b24..092534a1 100644 --- a/src/mcp/README.md +++ b/src/mcp/README.md @@ -41,7 +41,7 @@ The `BrainyMCPService` has been refactored to separate the core functionality fr ### In Any Environment (Browser, Node.js, Server) ```typescript -import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraft/brainy' +import { Brainy, BrainyMCPAdapter, MCPAugmentationToolset } from '@soulcraftlabs/brainy' // Create a Brainy instance const brainyData = new Brainy() @@ -81,7 +81,7 @@ const toolResponse = await toolset.handleRequest({ ### In Browser Environment (Core Functionality Only) ```typescript -import { Brainy, BrainyMCPService } from '@soulcraft/brainy' +import { Brainy, BrainyMCPService } from '@soulcraftlabs/brainy' // Create a Brainy instance const brainyData = new Brainy() diff --git a/src/plugin.ts b/src/plugin.ts index bfdc403a..b1aef8e0 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -22,7 +22,7 @@ import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js' // Re-export the provider contracts that already live closer to their // implementations so a plugin author (Cor) can import the *entire* -// provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`. +// provider surface from one stable entrypoint: `@soulcraftlabs/brainy/plugin`. export type { ColumnStoreProvider } from './indexes/columnStore/types.js' export type { AggregationProvider, @@ -41,7 +41,7 @@ export interface BrainyPlugin { name: string /** - * Optional semver range of `@soulcraft/brainy` this plugin supports + * Optional semver range of `@soulcraftlabs/brainy` this plugin supports * (e.g. `'>=8.0.0 <9.0.0'` or `'^8.0.0'`). When set and the running brainy is * OUTSIDE the range, `init()` THROWS rather than silently falling back to the * default JS engine. This is the version-coupling guard for the native diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 63356828..d9934c3b 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -215,7 +215,7 @@ export interface ScoreExplanation { * * @example * ```ts - * declare module '@soulcraft/brainy' { + * declare module '@soulcraftlabs/brainy' { * interface SubtypeRegistry { * // For NounType.Person, subtype 'employee': * 'person:employee': { employeeId: string; department: string } diff --git a/src/types/reservedFields.ts b/src/types/reservedFields.ts index 15b585c5..ce2108f8 100644 --- a/src/types/reservedFields.ts +++ b/src/types/reservedFields.ts @@ -65,7 +65,7 @@ * | `_rev` | system-managed revision counter — pass `ifRev` to `update()` for CAS | * * @example - * import { RESERVED_ENTITY_FIELDS } from '@soulcraft/brainy' + * import { RESERVED_ENTITY_FIELDS } from '@soulcraftlabs/brainy' * const isReserved = (key: string) => * (RESERVED_ENTITY_FIELDS as readonly string[]).includes(key) */ diff --git a/src/utils/brainyTypes.ts b/src/utils/brainyTypes.ts index 7db469bb..a008a5e6 100644 --- a/src/utils/brainyTypes.ts +++ b/src/utils/brainyTypes.ts @@ -6,7 +6,7 @@ * * @example * ```typescript - * import { BrainyTypes } from '@soulcraft/brainy' + * import { BrainyTypes } from '@soulcraftlabs/brainy' * * // Get all available types * const nounTypes = BrainyTypes.nouns // ['Person', 'Organization', ...] diff --git a/src/utils/version.ts b/src/utils/version.ts index d616cee3..f302eae0 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -1,6 +1,6 @@ /** * @module utils/version - * @description Resolves the running `@soulcraft/brainy` package version. Brainy 8.0 + * @description Resolves the running `@soulcraftlabs/brainy` package version. Brainy 8.0 * targets Node-like runtimes only (Node.js, Bun, Deno — all expose `node:fs`), so the * version is read **synchronously** from `package.json` on first call and cached. * diff --git a/tests/unit/brainy/migration-deference.test.ts b/tests/unit/brainy/migration-deference.test.ts index 31b9b216..b5817c3d 100644 --- a/tests/unit/brainy/migration-deference.test.ts +++ b/tests/unit/brainy/migration-deference.test.ts @@ -15,7 +15,7 @@ * - Hook 2: the public `brain.stampBrainFormat()` the provider calls once its * background migration has verified-and-swapped, authoring the shared * `_system/brain-format.json` marker. - * - Hook 3: the marker module is re-exported at `@soulcraft/brainy/brain-format` + * - Hook 3: the marker module is re-exported at `@soulcraftlabs/brainy/brain-format` * so cor reads the SAME `EXPECTED_INDEX_EPOCH` / `CURRENT_DATA_FORMAT` constants * (single source of truth, no duplicated value). * @@ -242,7 +242,7 @@ describe('rc.8 no-freeze migration deference (isMigrating / stampBrainFormat / b // --- Hook 3: marker module export ---------------------------------------- it('the brain-format marker module exports the compiled epoch + data-format constants', () => { - // cor imports these from '@soulcraft/brainy/brain-format' (Hook 3) so both + // cor imports these from '@soulcraftlabs/brainy/brain-format' (Hook 3) so both // sides share ONE source of truth — no duplicated constant to drift. // Epoch 3: the namespace-law key split (bare user keys · literal // 'system.' scalars, 2026-08-03) — every brain rebuilds onto the From 384f4b6b9c908baf1e6319f56427be1b50373353 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 17:10:30 -0700 Subject: [PATCH 041/150] chore(release): 10.4.3 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d13a2d66..56757f1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2...v10.4.3) (2026-08-27) + +- Merge branch 'next/open-brainy-rename' (a58372f0) +- chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83) +- docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24) + + ### [10.4.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) - docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) diff --git a/package-lock.json b/package-lock.json index 9cc3c6b0..4d247780 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.2", + "version": "10.4.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.2", + "version": "10.4.3", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 2312fcb0..bb6b5a47 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.2", + "version": "10.4.3", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 38c3397b600e776fcd737445a996a6cc37d2f315 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 17:26:44 -0700 Subject: [PATCH 042/150] =?UTF-8?q?docs:=20repository=20links=20point=20at?= =?UTF-8?q?=20soulcraftlabs/open-brainy=20=E2=80=94=20the=20soulcraft/brai?= =?UTF-8?q?ny=20path=20becomes=20the=20native=20engine's=20repo=20tonight?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 38 +++++++++++++++++++------------------- CONTRIBUTING.md | 4 ++-- README.md | 2 +- RELEASES.md | 2 +- scripts/release.sh | 4 ++-- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56757f1c..c7790837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,19 +2,19 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.4.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2...v10.4.3) (2026-08-27) +### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27) - Merge branch 'next/open-brainy-rename' (a58372f0) - chore: rename to @soulcraftlabs/brainy for Open Brainy on The Source (a99b1e83) - docs(releases): 10.4.3 — Open Brainy's first release under the new name, same engine as 10.4.2; The Source is the one registry (9f248b24) -### [10.4.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) +### [10.4.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2-rc.1...v10.4.2) (2026-08-27) - docs(releases): 10.4.1 and 10.4.2 consumer notes; 10.4.2 is the last MIT release under this name, Open Brainy continues at @soulcraftlabs/brainy (a082e0ef) -### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) +### [10.4.2-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) - Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) - fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) @@ -31,23 +31,23 @@ All notable changes to this project will be documented in this file. See [standa - chore(release): 10.4.1-rc.1 (7870dc40) -### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26) +### [10.4.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) - docs(guide): the docs pipeline publishes through the ingest API — the separate deploy step is retired (21e506e8) -### [10.4.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) +### [10.4.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.4...v10.4.0) (2026-08-26) - docs(releases): the 10.4.0 entry catches up to the late trains — repair routing, the vector ledger and open-gate leg, the loud config guard, the JSON-safe crossing (834149ed) -### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) +### [10.4.0-rc.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) - feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) -### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) +### [10.4.0-rc.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) - fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) - Merge branch 'worktree-agent-ad3aff0dffd17a6eb' (f14da34b) @@ -56,7 +56,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(open-path): init never gates on the embedding model; open goes concurrent; slow opens narrate (96624f40) -### [10.4.0-rc.2](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) +### [10.4.0-rc.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.0-rc.1...v10.4.0-rc.2) (2026-08-25) - test(readiness): the report helper's clock freezes — two independently-built reports compared across a millisecond tick made the plant lane red (39b916a3) - feat(repair): a heal:'repair' verdict routes to the provider's own incremental repair() (553e0d97) @@ -67,7 +67,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(health): the gate reads the named report — reads refuse loudly, never rebuild; open serves before it returns; the ceremony door (f8f64780) -### [10.4.0-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) +### [10.4.0-rc.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.1...v10.4.0-rc.1) (2026-08-24) - ci(publish): the home dist-tag follows the version — a prerelease publishes under 'rc' and never moves 'latest' (a1376e4a) - chore(release): --source-only — a home-only prerelease mode (The Source, never the storefront) (dcbad176) @@ -80,13 +80,13 @@ All notable changes to this project will be documented in this file. See [standa - ci(gate): the machine-health preflight and the truncation verdict guard (1e046aa1) -### [10.3.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.3.0...v10.3.1) (2026-08-18) +### [10.3.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.3.0...v10.3.1) (2026-08-18) - docs(releases): the 10.3.1 consumer entry — the fold that behaves (900cc895) - fix(recovery): the fold streams and narrates; the checkpoint chain arms at the flip (ed7d1db9) -### [10.3.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.2.0...v10.3.0) (2026-08-18) +### [10.3.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.2.0...v10.3.0) (2026-08-18) - docs(releases): the 10.3.0 consumer entry — the trust-and-provenance release (97d75649) - fix(locks): the fence keys ownership on pid+hostname — a same-process re-open never fences its predecessor (0991cf28) @@ -95,14 +95,14 @@ All notable changes to this project will be documented in this file. See [standa - feat(log): system commits carry their origin; the attested per-id reconcile door (9ac9e706) -### [10.2.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.1.0...v10.2.0) (2026-08-17) +### [10.2.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.1.0...v10.2.0) (2026-08-17) - docs(releases): the 10.2.0 consumer entry — adoption completes in one call (97538e1f) - ci: the correctness plant runs integration + conformance on every push — a release never waits on a second machine (b17fdc8e) - fix(adoption): the baseline backfill runs to completion — one call adopts a pre-log baseline of any size (a5a18838) -### [10.1.0](https://source.soulcraft.com/soulcraft/brainy/compare/v10.0.0...v10.1.0) (2026-08-13) +### [10.1.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.0.0...v10.1.0) (2026-08-13) - docs(releases): the 10.1.0 consumer entry — bounded recovery, restore founding, the two write-path cures (7d3c8696) - fix(restore): a restore is an unclean event — the swap runs quiesced and the snapshot's durability stamps never survive it (9ca80667) @@ -111,7 +111,7 @@ All notable changes to this project will be documented in this file. See [standa - feat(query): the sparse-store cut — where on a never-carried field serves operator truth, never a refusal (7b67db4d) -### [10.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v9.0.0...v10.0.0) (2026-08-12) +### [10.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v9.0.0...v10.0.0) (2026-08-12) - fix(adoption): the baseline backfill cures hydration-law drift — existing brains reach the crash-safe default with zero operator steps (25f0dd96) - fix(adoption): the reserved-root mint exemption — int 0 is legitimate for exactly one id (2abe8b38) @@ -143,7 +143,7 @@ All notable changes to this project will be documented in this file. See [standa - test: version-coupling pins go major-agnostic — the 8.x literals broke at the 9.0.0 bump while the coupling law itself behaved correctly (8a6807e8) -### [9.0.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.11.0...v9.0.0) (2026-08-04) +### [9.0.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.11.0...v9.0.0) (2026-08-04) - docs: 9.0 namespace-migration guide — the simple story + the mechanical sweep checklist, published for humans and tooling alike (61ab9db2) - fix(release): storefront leg republishes CI's exact forge artifact — byte-identity by construction, verified by cross-registry shasum before the ceremony reports success (d89df2ed) @@ -178,7 +178,7 @@ All notable changes to this project will be documented in this file. See [standa - feat: scanFacts liveness contract — first batch or loud failure within a documented bound (f8e6da2b) -### [8.11.0](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.11.0) (2026-07-27) +### [8.11.0](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.11.0) (2026-07-27) - docs: the last two archived-host links point home (91ef1c8b) - feat: includeHidden — export carries every visibility tier for migration-grade canon completeness (63c1eeb9) @@ -187,19 +187,19 @@ All notable changes to this project will be documented in this file. See [standa - ci: run the pipeline on the forge (999d0ebb) -### [8.10.3](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.2...v8.10.3) (2026-08-03) +### [8.10.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.2...v8.10.3) (2026-08-03) - docs: dedupe the 8.10.2 release-notes entry the cherry doubled onto the branch (8c956608) - fix: user metadata named 'level' is a real field everywhere — the engine-internal node layer no longer shadows it in sort/filter/aggregation, and the indexing views stop stamping a phantom 0 into its column; index epoch 2 rebuilds existing brains at first open (958a0859) -### [8.10.2](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.1...v8.10.2) (2026-07-29) +### [8.10.2](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.1...v8.10.2) (2026-07-29) - docs: 8.10.2 consumer release notes — update() write granularity, PathResolver idle-log fix, graph-lsm key recognition (a0123b5b) - fix: metadata-only update() never rewrites the noun record — the unconditional whole-vector save turned per-entity stat touches into full rewrites+fsync, amplifying read-heavy sweeps into disk saturation on a production deployment (5b65eb82) -### [8.10.1](https://source.soulcraft.com/soulcraft/brainy/compare/v8.10.0...v8.10.1) (2026-07-24) +### [8.10.1](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v8.10.0...v8.10.1) (2026-07-24) - refactor: remove the orphaned transaction-result type left behind by the dead-path removal (edf123a5) - fix: warm() metadata surface routes through the active provider (warm hook added to the metadata contract); add maintenanceDebt() observability surface (5b2cbf74) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d277091d..50860cb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ may find elsewhere in the repo's history. ## Where the project lives -The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraft/brainy**. +The source of truth is a self-hosted forge: **source.soulcraft.com/soulcraftlabs/open-brainy**. It's anonymously readable and cloneable — no account needed to browse, clone, or build. @@ -31,7 +31,7 @@ fine) to talk through the approach saves everyone rework. ## Development setup ```bash -git clone https://source.soulcraft.com/soulcraft/brainy.git +git clone https://source.soulcraft.com/soulcraftlabs/open-brainy.git cd brainy npm install npm run build diff --git a/README.md b/README.md index 47a9123a..762c9ec3 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

Package on The Source Repository - CI + CI Documentation MIT License TypeScript diff --git a/RELEASES.md b/RELEASES.md index b89beba0..4d716efa 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,7 +1,7 @@ # @soulcraft/brainy — Release Notes for Consumers This file is the **quick reference for downstream sessions** tracking Brainy changes. -Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraft/brainy/releases +Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases **How to use:** Brainy is the underlying data engine for downstream applications. Read this when: - Upgrading `@soulcraft/brainy` in your application diff --git a/scripts/release.sh b/scripts/release.sh index be1d6d6b..08293e3a 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,7 +154,7 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraft/brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) ${COMMITS} " @@ -264,4 +264,4 @@ echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━ echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" -echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraft/brainy/releases/tag/v${NEW_VERSION}${NC}" +echo -e "🏠 The Source: ${BLUE}https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${NEW_VERSION}${NC}" From e652162c1fe2e86cbdd0441094938e4717f10829 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:17:20 -0700 Subject: [PATCH 043/150] fix(storage): a clean close is recorded, and the writer lock is always given up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production restart made this necessary: a service stopped with exit code 0, having awaited close() on every pooled brain, and its next boot announced "Overwriting stale writer lock ... appears dead" for every store it owned. Nothing had crashed. "The recorded pid is gone" is equally true of an orderly restart and of a crash, so the verdict could not tell an operator which one they had — and when the OS recycles a pid it fails the other way, refusing to open a store whose writer died days ago. Three changes, all at the law: - close() is two parts, and the second is unconditional. The durable steps (flush, markers, component close, plugin deactivate, buffer drain) move to closeDurableSteps(); the terminal releases — the flush-request watcher, the WRITER LOCK, the VFS timers, the terminal `closed` flag — always run. The original failure is narrated with what it costs the next open, then rethrown. - releaseWriterLock() writes a CLEAN-CLOSE RECORD (`locks/_writer.close`) naming the lock generation it released; the next claim consumes it, so a record can never vouch for a later crash. An open reads the record instead of guessing: recorded → nothing to recover; absent → say so, and name the crash recovery this open will now run. - The signal path stops failing in a batch. It was one try around a loop over every open brain, so the first instance whose flush rejected stranded every remaining brain's lock and markers — at exit code 0. Now: per-instance isolation, the generation store's close (the clean-shutdown marker, without which the next open folds the whole log) is part of shutdown, the lock is given up in a finally, and the handler no longer calls process.exit() when the host application has its own signal handler — that race truncated the host's own close() mid-flight. Pins: tests/integration/writer-lock-clean-close.test.ts — completed close leaves no lock and a consumed-once record with a silent reopen; a failing durable step still releases and still rethrows; SIGKILL leaves the lock with no record and the reopen names the crash; a host SIGTERM handler runs to completion. Branch plan (10 lines): 1. writer lock: clean-close record + always-release close [this commit] 2. open narration: an always-on channel; production clamps prodLog to ERROR, which is why a three-minute open printed nothing 3. open narration: per-phase lines as each phase ENDS, with progress cadence 4. measure both real-store fixtures on the box, before/after 5. move the generation-log fold out of the foreground where the serving law allows; durable resumable progress marker 6. same for the VFS bootstrap 7. counts: a legacy container-rule ledger must not keep serving wrong denominators; counts.json written atomically 8. counts pin with scar directories; two copies of one archive agree 9. docs/canonical-layout-ratification.md — 12 facts confirmed/corrected 10. report: MEASURED before/after, findings, and whether this is 10.4.4 --- src/brainy.ts | 322 ++++++++++++------ src/storage/adapters/fileSystemStorage.ts | 147 +++++++- src/storage/baseStorage.ts | 30 ++ .../writer-lock-clean-close.test.ts | 250 ++++++++++++++ 4 files changed, 641 insertions(+), 108 deletions(-) create mode 100644 tests/integration/writer-lock-clean-close.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 966d3188..957920f5 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1856,76 +1856,112 @@ export class Brainy implements BrainyInterface { * NOTE: Registers globally (once for all instances) to avoid MaxListenersExceededWarning */ private registerShutdownHooks(): void { + /** + * The signal-path shutdown. THREE LAWS, each written by a production + * shutdown that looked clean and wasn't: + * + * 1. PER-INSTANCE ISOLATION. This used to be one `try` around a loop over + * every open brain: the first instance whose flush rejected aborted the + * loop, so every remaining brain kept its writer lock and its unwritten + * markers — and the process still exited 0. A pool of brains failed in + * a batch, not one at a time. + * 2. THE MARKER IS PART OF SHUTDOWN. Flushing the indexes without closing + * the generation store leaves the clean-shutdown marker unwritten, so + * the NEXT open reads the store as crashed and folds the whole + * generation log — measured in tens of seconds on a real store, paid on + * every restart, after a shutdown the operator saw exit 0. + * 3. THE LOCK IS ALWAYS GIVEN UP. In a `finally`, per instance: a process + * on its way out holds nothing. + */ const flushOnShutdown = async () => { console.log('Shutdown signal received - flushing pending data...') - try { - let flushedCount = 0 - for (const instance of Brainy.instances) { - if (instance.initialized) { - // Flush all buffered data, then close to release resources (timers, handles) - await Promise.all([ - (async () => { - if (instance.storage && typeof instance.storage.flushCounts === 'function') { - await instance.storage.flushCounts() - } - })(), - (async () => { - if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { - await instance.metadataIndex.flush() - } - })(), - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { - await instance.graphIndex.flush() - } - })(), - (async () => { - if (instance.index && typeof instance.index.flush === 'function') { - await instance.index.flush() - } - })() - ]) - // Close components to stop timers that would prevent clean process exit - await Promise.all([ - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { - await instance.graphIndex.close() - } - })(), - (async () => { - const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks - if (index && typeof index.close === 'function') { - await index.close() - } - })(), - (async () => { - const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks - if (metadataIndex && typeof metadataIndex.close === 'function') { - await metadataIndex.close() - } - })(), - // Release the writer lock so a successor process can take over. - // No-op for readers and for backends without locking. - (async () => { - if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { - await instance.storage.releaseWriterLock() - } - })(), - // Stop the flush-request watcher to release its interval timer. - (async () => { - if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { - instance.storage.stopFlushRequestWatcher() - } - })(), - ]) - flushedCount++ + let flushedCount = 0 + let failedCount = 0 + // Snapshot: close() splices Brainy.instances while we iterate. + for (const instance of [...Brainy.instances]) { + if (!instance.initialized) continue + try { + // Flush all buffered data (parallel across components, this brain only). + await Promise.all([ + (async () => { + if (instance.storage && typeof instance.storage.flushCounts === 'function') { + await instance.storage.flushCounts() + } + })(), + (async () => { + if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { + await instance.metadataIndex.flush() + } + })(), + (async () => { + if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { + await instance.graphIndex.flush() + } + })(), + (async () => { + if (instance.index && typeof instance.index.flush === 'function') { + await instance.index.flush() + } + })() + ]) + + // Close the generation store: persists the counter, advances the + // fold checkpoint, and stamps the clean-shutdown marker LAST — the + // one step that decides whether the next open adopts or folds. Law 2. + if (instance.generationStore && !instance.isReadOnly) { + await instance.generationStore.close() + } + + // Close components to stop timers that would prevent clean process exit + await Promise.all([ + (async () => { + if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { + await instance.graphIndex.close() + } + })(), + (async () => { + const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks + if (index && typeof index.close === 'function') { + await index.close() + } + })(), + (async () => { + const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks + if (metadataIndex && typeof metadataIndex.close === 'function') { + await metadataIndex.close() + } + })() + ]) + flushedCount++ + } catch (error) { + failedCount++ + console.error('Failed to flush one Brainy instance on shutdown:', error) + } finally { + // Law 3 — the lock and the watcher go regardless. + try { + if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { + instance.storage.stopFlushRequestWatcher() + } + } catch (error) { + console.error('Failed to stop the flush-request watcher on shutdown:', error) + } + try { + if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { + await instance.storage.releaseWriterLock() + } + } catch (error) { + console.error('Failed to release the writer lock on shutdown:', error) } } - if (flushedCount > 0) { - console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) - } - } catch (error) { - console.error('Failed to flush on shutdown:', error) + } + if (flushedCount > 0) { + console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) + } + if (failedCount > 0) { + console.error( + `${failedCount} Brainy instance${failedCount > 1 ? 's' : ''} did not complete shutdown — ` + + `their writer locks were released, but their next open will run crash recovery.` + ) } } @@ -1933,13 +1969,32 @@ export class Brainy implements BrainyInterface { // kept as statics so the last live instance's close() can deregister them // — the signal handles they hold are ref'd and would otherwise keep the // process alive forever after every brain is closed. + /** + * Exit the process ONLY when Brainy is the sole handler for this signal. + * + * Registering a signal listener suppresses Node's default terminate + * behaviour, so a library that attaches one must either exit or be sure + * someone else will. Brainy attaching one AND exiting was the wrong half + * of that choice for every host application with its own graceful + * shutdown: both handlers run concurrently, and whichever finishes first + * wins — a library flush finishing before an application's close() + * terminated that close mid-flight, at exit code 0, with locks and + * markers unwritten. When the host has its own handler (listener count + * above our own), the host owns the exit; Brainy only makes its data + * durable and steps aside. + */ + const exitIfSoleShutdownOwner = (signal: 'SIGTERM' | 'SIGINT'): void => { + if (process.listenerCount(signal) <= 1) { + process.exit(0) + } + } Brainy.sigtermListener = async () => { await flushOnShutdown() - process.exit(0) + exitIfSoleShutdownOwner('SIGTERM') } Brainy.sigintListener = async () => { await flushOnShutdown() - process.exit(0) + exitIfSoleShutdownOwner('SIGINT') } Brainy.beforeExitListener = async () => { // Self-deregister FIRST: Node re-emits 'beforeExit' after every event- @@ -18877,12 +18932,105 @@ export class Brainy implements BrainyInterface { } /** - * Close and cleanup + * @description Close and clean up: flush every buffered component, stamp + * the durability markers, release resources, then give up the writer lock. * - * Now flushes HNSW dirty nodes before closing - * This ensures deferred persistence mode data is saved + * TWO PARTS, AND THE SECOND IS UNCONDITIONAL. Everything that persists data + * runs in {@link closeDurableSteps}; the terminal releases — the flush-request + * watcher, the WRITER LOCK, the VFS timers, and the terminal `closed` flag — + * run whether those steps succeeded or not, in a `finally`. A close that + * threw halfway used to strand the writer lock on disk with this process's + * (soon dead) pid in it, so the next boot of every affected store announced + * `Overwriting stale writer lock … appears dead` after an orderly exit and + * an operator had to decide whether their database had crashed. A closed + * brain holds no lock — there is no failure for which the opposite is the + * safer answer. + * + * The original failure is never swallowed: it is narrated with what it costs + * the next open, then rethrown to the caller. + * @returns Nothing. + * @throws The first failure from the durable close steps, after the + * terminal releases have run. */ async close(): Promise { + let closeFailure: unknown = null + try { + await this.closeDurableSteps() + } catch (error) { + closeFailure = error + } + + // ---- TERMINAL RELEASES: always, even after a failure above ---- + + // Stop the cross-process flush-request watcher (no-op if never started). + try { + if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { + this.storage.stopFlushRequestWatcher() + } + } catch (error) { + console.warn('[Brainy] close: stopping the flush-request watcher failed:', error) + } + + // Release the writer lock. Runs after the metadata buffer drain in + // closeDurableSteps() — otherwise a pending write could land after a + // successor writer claimed the lock — and runs even if that drain threw: + // holding a lock from a process that is about to exit locks the store's + // next boot out of a clean verdict. + try { + if (this.storage && typeof this.storage.releaseWriterLock === 'function') { + await this.storage.releaseWriterLock() + } + } catch (error) { + console.warn('[Brainy] close: releasing the writer lock failed:', error) + } + + // Shut down the VFS: stops its background maintenance interval and the + // PathResolver's — both are ref'd timers that would keep the process + // alive after the last brain closes (consumer-reported hang). + try { + if (this._vfs) { + await this._vfs.close() + } + } catch (error) { + console.warn('[Brainy] close: VFS shutdown failed:', error) + } + + this.initialized = false + // close() is terminal: block lazy re-initialization on any subsequent + // operation (ensureInitialized() throws once this is set). Set even when + // the durable steps failed — a half-closed brain must not keep serving. + this.closed = true + + // Drop this instance from the global registry, and when it was the last + // one, deregister the global shutdown hooks — their ref'd signal handles + // would otherwise keep the process alive after every brain is closed. + const instanceIndex = Brainy.instances.indexOf(this) + if (instanceIndex !== -1) { + Brainy.instances.splice(instanceIndex, 1) + } + Brainy.deregisterShutdownHooksIfIdle() + + if (closeFailure !== null) { + console.error( + `[Brainy] close FAILED partway: ` + + `${closeFailure instanceof Error ? closeFailure.message : String(closeFailure)}\n` + + ` This brain is closed and holds no writer lock, but the clean-shutdown ` + + `marker may not have been written — the next open will run crash recovery ` + + `(a generation-log fold) and report its wall.` + ) + throw closeFailure + } + } + + /** + * @description The durable half of {@link close}: flush every component, + * persist the generation counter and its markers, close the components, + * deactivate plugins, drain the metadata write buffer. Separated from + * `close()` so the terminal releases there can run in a `finally` — see that + * method's contract. + * @returns Nothing. + */ + private async closeDurableSteps(): Promise { // Persistence cadence teardown: no background flush may fire after close // begins (close() runs its own final flush). if (this._persistIdleTimer) { @@ -19033,38 +19181,6 @@ export class Brainy implements BrainyInterface { } } - // Stop the cross-process flush-request watcher (no-op if never started). - if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { - this.storage.stopFlushRequestWatcher() - } - - // Release the writer lock (no-op for readers and for backends that don't - // hold a lock). Must run after the metadata buffer drain — otherwise a - // pending write could land after a successor writer claimed the lock. - if (this.storage && typeof this.storage.releaseWriterLock === 'function') { - await this.storage.releaseWriterLock() - } - - // Shut down the VFS: stops its background maintenance interval and the - // PathResolver's — both are ref'd timers that would keep the process - // alive after the last brain closes (consumer-reported hang). - if (this._vfs) { - await this._vfs.close() - } - - this.initialized = false - // close() is terminal: block lazy re-initialization on any subsequent - // operation (ensureInitialized() throws once this is set). - this.closed = true - - // Drop this instance from the global registry, and when it was the last - // one, deregister the global shutdown hooks — their ref'd signal handles - // would otherwise keep the process alive after every brain is closed. - const instanceIndex = Brainy.instances.indexOf(this) - if (instanceIndex !== -1) { - Brainy.instances.splice(instanceIndex, 1) - } - Brainy.deregisterShutdownHooksIfIdle() } } diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 4f2a43b0..a01c2015 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -14,7 +14,8 @@ import { StorageBatchConfig, SYSTEM_DIR, STATISTICS_KEY, - WriterLockInfo + WriterLockInfo, + WriterCloseRecord } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' @@ -99,6 +100,13 @@ export class FileSystemStorage extends BaseStorage { // timer rewrites the lock every 10s so stale-lock detection can tell a dead // writer from a slow one. The constant name matches the file path used. private static readonly WRITER_LOCK_FILE = '_writer.lock' + /** + * The clean-close record at `locks/_writer.close` (see + * {@link WriterCloseRecord}). Written when the lock is released, consumed by + * the next claim, so an open can distinguish "the previous writer left" from + * "the previous writer died" without inferring either from a pid. + */ + private static readonly WRITER_CLOSE_FILE = '_writer.close' private static readonly WRITER_HEARTBEAT_MS = 10_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout @@ -1902,11 +1910,24 @@ export class FileSystemStorage extends BaseStorage { rootDir: this.rootDir } await this.writeFileAtomic(lockFile, JSON.stringify(info, null, 2)) + await this.clearWriterCloseRecord() this.installWriterLock(info) return info } - const stale = !options?.force && (await this.isWriterLockStale(existing)) + // THE CLEAN-CLOSE RECORD IS CONSULTED FIRST (see WriterCloseRecord). + // A lock file whose release was RECORDED is bookkeeping left behind by + // an orderly shutdown, not evidence of a crash — take it over calmly + // and say so. Only when no record vouches for this lock do we fall + // back to inferring liveness from the pid, and then we say THAT + // honestly too: an unrecorded lock means the writer did not complete + // its close, so the store was not closed cleanly and this open pays + // recovery. + const closeRecord = await this.readWriterCloseRecord() + const releasedCleanly = + closeRecord !== null && this.closeRecordVouchesFor(closeRecord, existing) + const stale = + releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing))) if (!options?.force && !stale) { // Consumer-facing error contract: callers detect this case via // err.code and read the holder's details from err.lockInfo. @@ -1917,8 +1938,16 @@ export class FileSystemStorage extends BaseStorage { options?.force ? `[brainy] Force-overwriting writer lock for ${this.rootDir} ` + `(was held by PID ${existing.pid} on ${existing.hostname}).` - : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + - `(PID ${existing.pid} on ${existing.hostname} appears dead).` + : releasedCleanly + ? `[brainy] Clearing the leftover writer lock for ${this.rootDir} — ` + + `PID ${existing.pid} on ${existing.hostname} RELEASED it cleanly at ` + + `${closeRecord!.closedAt} but could not remove the file. ` + + `Nothing to recover.` + : `[brainy] Overwriting stale writer lock for ${this.rootDir} ` + + `(PID ${existing.pid} on ${existing.hostname} is gone and left NO ` + + `clean-close record — that writer did not finish closing, so this ` + + `store was not closed cleanly; open will run crash recovery and ` + + `report its wall).` ) // Takeover: verify the file still holds the lock we judged (a live // successor may have claimed meanwhile), then remove it and fall @@ -1972,6 +2001,12 @@ export class FileSystemStorage extends BaseStorage { await fs.promises.unlink(claimTmp).catch(() => {}) } + // CONSUME the previous writer's clean-close record. It described the + // lock generation that just ended; leaving it in place would let it + // vouch for OUR lock if this process later dies without closing — + // turning a real crash into a "closed cleanly" verdict. One unlink. + await this.clearWriterCloseRecord() + this.installWriterLock(info) return info } @@ -2095,13 +2130,27 @@ export class FileSystemStorage extends BaseStorage { return } const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) + const released = this.writerLockInfo try { // Only delete if we still own it — avoid clobbering a successor that // claimed the lock via force-override. const current = await this.readWriterLock() - if (current && current.pid === this.writerLockInfo.pid && current.hostname === this.writerLockInfo.hostname) { + const ours = + current === null || + (current.pid === released.pid && current.hostname === released.hostname) + if (current && ours) { await fs.promises.unlink(lockFile) } + // THE CLEAN-CLOSE RECORD (see WriterCloseRecord). Written whenever this + // instance gives up a lock nobody else has taken — the unlink above + // having succeeded OR the file already being gone. The next open reads + // it instead of guessing from pid liveness: a recorded release is an + // orderly shutdown, an absent record is a writer that never finished + // closing. Not written when a successor holds the lock: our release is + // then a no-op and a record would slander their live lock. + if (ours) { + await this.writeWriterCloseRecord(released) + } } catch (err: any) { if (err.code !== 'ENOENT') { console.warn('[brainy] Failed to release writer lock file:', err) @@ -2111,6 +2160,94 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * @description Read the clean-close record at `locks/_writer.close`, or + * `null` when it is absent or unparseable. A torn record is treated as + * absent — the conservative direction, since an unreadable record can + * vouch for nothing. + * @returns The record, or null. + */ + public async readWriterCloseRecord(): Promise { + await this.ensureInitialized() + const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) + try { + const raw = await fs.promises.readFile(recordFile, 'utf-8') + const parsed = JSON.parse(raw) as WriterCloseRecord + if ( + typeof parsed?.pid !== 'number' || + typeof parsed?.hostname !== 'string' || + typeof parsed?.startedAt !== 'string' || + typeof parsed?.closedAt !== 'string' + ) { + return null + } + return parsed + } catch (err: any) { + if (err.code === 'ENOENT') return null + return null + } + } + + /** + * @description Whether a clean-close record describes the very lock + * generation `lock` represents. The match is pid + hostname + `startedAt`: + * `startedAt` is the lock generation's identity, so a record can never + * vouch for a LATER lock taken by the same pid on the same host (the + * same-process re-open path mints a fresh `startedAt`). + * @param record - The clean-close record read from disk. + * @param lock - The lock file's contents. + */ + private closeRecordVouchesFor(record: WriterCloseRecord, lock: WriterLockInfo): boolean { + return ( + record.pid === lock.pid && + record.hostname === lock.hostname && + record.startedAt === lock.startedAt + ) + } + + /** + * @description Write the clean-close record for a lock this instance just + * released. Atomic (temp + rename) so a concurrent opener never reads half + * a record. A failure here costs the next open nothing but the honest + * fallback (pid liveness), so it warns rather than failing the close. + * @param released - The lock info this instance held. + */ + private async writeWriterCloseRecord(released: WriterLockInfo): Promise { + const record: WriterCloseRecord = { + pid: released.pid, + hostname: released.hostname, + startedAt: released.startedAt, + closedAt: new Date().toISOString(), + version: released.version + } + const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) + try { + await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2)) + } catch (err) { + console.warn( + `[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` + + `the next open will fall back to pid liveness and may report this orderly ` + + `shutdown as a crash:`, + err + ) + } + } + + /** + * @description Remove the clean-close record. Called by every successful + * lock claim so a record never outlives the lock generation it describes. + */ + private async clearWriterCloseRecord(): Promise { + const recordFile = path.join(this.lockDir, FileSystemStorage.WRITER_CLOSE_FILE) + try { + await fs.promises.unlink(recordFile) + } catch (err: any) { + if (err.code !== 'ENOENT') { + console.warn('[brainy] Failed to clear the writer clean-close record:', err) + } + } + } + public override async readWriterLock(): Promise { await this.ensureInitialized() const lockFile = path.join(this.lockDir, FileSystemStorage.WRITER_LOCK_FILE) diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 5510f93b..f518e68f 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -125,6 +125,36 @@ export interface WriterLockInfo { rootDir?: string // Convenience for log lines / error messages } +/** + * THE CLEAN-CLOSE RECORD. Written by `releaseWriterLock()` at the instant it + * gives up the writer lock, naming the lock identity it released. The next + * `acquireWriterLock()` reads it and can then say — from a RECORD, not from a + * guess — whether the previous writer left on purpose. + * + * Why a record and not PID liveness: "the recorded PID is no longer alive" is + * true of every orderly restart AND of every crash, so the two were reported + * identically ("appears dead") and neither could be trusted. Worse, the same + * inference fails the other way when the operating system RECYCLES the pid — + * a live unrelated process makes a long-dead writer's lock look held, and the + * store refuses to open naming a pid that was never Brainy. A record settles + * both: matched → the previous writer closed cleanly, nothing to recover; + * absent → say so, and name what recovery the open will now run. + * + * Lifecycle: written at release, consumed (deleted) by the next successful + * lock claim — a record must never outlive the lock generation it describes, + * or it would vouch for a later crash. + */ +export interface WriterCloseRecord { + pid: number + hostname: string + /** `startedAt` of the lock this close released — the identity match key. */ + startedAt: string + /** ISO timestamp at which the lock was released. */ + closedAt: string + /** Brainy version that performed the close. */ + version: string +} + /** * FNV-1a hash returning a 2-char hex bucket (00-ff). * Distributes system keys across 256 sub-prefixes to avoid diff --git a/tests/integration/writer-lock-clean-close.test.ts b/tests/integration/writer-lock-clean-close.test.ts new file mode 100644 index 00000000..7d9c59d6 --- /dev/null +++ b/tests/integration/writer-lock-clean-close.test.ts @@ -0,0 +1,250 @@ +/** + * @module tests/integration/writer-lock-clean-close + * @description THE CLEAN-CLOSE CONTRACT for the writer lock. + * + * A production restart made this lane necessary: a service stopped with exit + * code 0, having awaited `close()` on every pooled brain, and its next boot + * announced `[brainy] Overwriting stale writer lock … appears dead` for every + * store it owned. "The pid is gone" is equally true of an orderly restart and + * of a crash, so the message could not tell an operator which one they had. + * + * The contract pinned here: + * 1. A completed close leaves NO lock file and DOES leave a clean-close + * record; the next open says nothing about staleness. + * 2. The next lock claim CONSUMES that record — it may never outlive the + * lock generation it describes, or a later crash would read as clean. + * 3. A close whose durable steps FAIL still releases the lock (and still + * rethrows the failure). + * 4. A killed process (SIGKILL, no close at all) leaves the lock behind with + * NO record, and the next open says exactly that — crash, recovery ahead. + * 5. A host application with its own SIGTERM handler is never force-exited + * out from under its own shutdown by Brainy's handler. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs' +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const REPO_ROOT = process.cwd() +const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx') + +function makeTempDir(): string { + return mkdtempSync(join(tmpdir(), 'brainy-clean-close-')) +} + +/** + * Write a child script to disk and start it under tsx. A file (not `tsx -e`) + * because the eval form compiles to CommonJS, which has no top-level await. + * The script imports Brainy by ABSOLUTE path, so its own dependency + * resolution still happens from inside the repository. + */ +function startChild(dir: string, body: string): ReturnType { + const scriptPath = join(dir, 'child-process.mts') + writeFileSync(scriptPath, body) + // `detached` puts the child in its own process GROUP: tsx runs the script in + // a grandchild process, and only a group-wide signal reaches the process + // that actually holds the writer lock. + return spawn(TSX, [scriptPath], { + cwd: REPO_ROOT, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true + }) +} + +/** Capture every console.warn/error line emitted while `fn` runs. */ +async function captureConsole(fn: () => Promise): Promise<{ result: T; lines: string[] }> { + const lines: string[] = [] + const origWarn = console.warn + const origError = console.error + const sink = (...args: unknown[]) => { + lines.push(args.map((a) => String(a)).join(' ')) + } + console.warn = sink as typeof console.warn + console.error = sink as typeof console.error + try { + const result = await fn() + return { result, lines } + } finally { + console.warn = origWarn + console.error = origError + } +} + +/** + * Run a child process that opens `dir`, writes one row, prints `READY`, and + * then waits forever. Resolves with the child once READY is seen. + */ +function spawnHoldingChild(dir: string): Promise<{ + child: ReturnType + output: () => string +}> { + const script = ` + import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) + await brain.init() + await brain.add({ data: 'row from the child', type: 'concept' }) + await brain.flush() + console.log('READY') + setInterval(() => {}, 1000) + ` + const child = startChild(dir, script) + let out = '' + child.stdout.on('data', (d) => { out += String(d) }) + child.stderr.on('data', (d) => { out += String(d) }) + return new Promise((resolvePromise, rejectPromise) => { + const timer = setTimeout(() => rejectPromise(new Error(`child never became READY:\n${out}`)), 120_000) + child.stdout.on('data', () => { + if (out.includes('READY')) { + clearTimeout(timer) + resolvePromise({ child, output: () => out }) + } + }) + child.on('exit', (code) => { + clearTimeout(timer) + if (!out.includes('READY')) rejectPromise(new Error(`child exited ${code} before READY:\n${out}`)) + }) + }) +} + +describe('writer lock — the clean-close contract', () => { + let dir: string + let brain: Brainy | null = null + + beforeEach(() => { dir = makeTempDir() }) + + afterEach(async () => { + if (brain) { + try { await brain.close() } catch { /* may already be closed */ } + brain = null + } + try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } + }) + + const lockPath = () => join(dir, 'locks', '_writer.lock') + const recordPath = () => join(dir, 'locks', '_writer.close') + + it('a completed close leaves no lock, leaves a record, and the reopen is silent about staleness', async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + expect(existsSync(lockPath())).toBe(true) + + await brain.add({ data: 'seed entity', type: NounType.Concept }) + await brain.flush() + await brain.close() + brain = null + + // 1. The lock is gone and the release is RECORDED. + expect(existsSync(lockPath())).toBe(false) + expect(existsSync(recordPath())).toBe(true) + const record = JSON.parse(readFileSync(recordPath(), 'utf-8')) + expect(record.pid).toBe(process.pid) + expect(typeof record.closedAt).toBe('string') + expect(typeof record.startedAt).toBe('string') + + // 2. The reopen says nothing about a stale lock. + const { result: reopened, lines } = await captureConsole(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + return next + }) + brain = reopened + expect(lines.filter((l) => /stale writer lock|appears dead/i.test(l))).toEqual([]) + + // 3. The claim CONSUMED the record — it must not outlive its lock generation. + expect(existsSync(recordPath())).toBe(false) + expect(existsSync(lockPath())).toBe(true) + }, 120_000) + + it('releases the writer lock even when a durable close step fails — and still rethrows', async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + await brain.add({ data: 'seed entity', type: NounType.Concept }) + await brain.flush() + expect(existsSync(lockPath())).toBe(true) + + // Inject a failure into a durable close step (the counts flush). + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const boom = new Error('injected: counts flush failed during close') + storage.flushCounts = async () => { throw boom } + + await expect(brain.close()).rejects.toThrow(/injected: counts flush failed/) + brain = null + + // The lock is released regardless: a process on its way out holds nothing. + expect(existsSync(lockPath())).toBe(false) + + // And the next writer opens without a stale-lock verdict. + const { lines } = await captureConsole(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + await next.close() + }) + expect(lines.filter((l) => /appears dead/i.test(l))).toEqual([]) + }, 120_000) + + it('a SIGKILLed writer leaves the lock with no record, and the next open names the crash', async () => { + const { child } = await spawnHoldingChild(dir) + expect(existsSync(lockPath())).toBe(true) + expect(existsSync(recordPath())).toBe(false) + + // Group-wide: the lock holder is tsx's grandchild, not the spawned pid. + process.kill(-(child.pid as number), 'SIGKILL') + await new Promise((r) => child.on('exit', () => r())) + // The grandchild's death is asynchronous with the wrapper's exit event. + await new Promise((r) => setTimeout(r, 500)) + + // The lock survives the kill — a dead process releases nothing. + expect(existsSync(lockPath())).toBe(true) + expect(existsSync(recordPath())).toBe(false) + + const { lines } = await captureConsole(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + await next.close() + }) + const verdict = lines.filter((l) => /Overwriting stale writer lock/i.test(l)) + expect(verdict.length).toBe(1) + // The verdict must name the ABSENT record and the recovery it implies — + // not merely that a pid is gone. + expect(verdict[0]).toMatch(/NO\s+clean-close record/i) + expect(verdict[0]).toMatch(/crash recovery/i) + }, 180_000) + + it("does not force-exit a host application that owns its own SIGTERM handler", async () => { + const script = ` + import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) + await brain.init() + await brain.add({ data: 'row from the host app', type: 'concept' }) + await brain.flush() + // The host application's OWN graceful shutdown, registered after Brainy's. + process.on('SIGTERM', async () => { + await new Promise((r) => setTimeout(r, 1500)) + console.log('APP-CLOSE-DONE') + process.exit(0) + }) + console.log('READY') + setInterval(() => {}, 1000) + ` + const child = startChild(dir, script) + let out = '' + child.stdout.on('data', (d) => { out += String(d) }) + child.stderr.on('data', (d) => { out += String(d) }) + await new Promise((r, reject) => { + const timer = setTimeout(() => reject(new Error(`child never became READY:\n${out}`)), 120_000) + child.stdout.on('data', () => { if (out.includes('READY')) { clearTimeout(timer); r() } }) + child.on('exit', () => { clearTimeout(timer); if (!out.includes('READY')) reject(new Error(`child died:\n${out}`)) }) + }) + + process.kill(-(child.pid as number), 'SIGTERM') + const code = await new Promise((r) => child.on('exit', (c) => r(c))) + expect(code).toBe(0) + // The host's own shutdown ran to completion — Brainy's handler did not + // exit the process out from under it. + expect(out).toContain('APP-CLOSE-DONE') + }, 180_000) +}) From afe08a1ff990ed451caad2a673f7147e59fc3567 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:19:55 -0700 Subject: [PATCH 044/150] feat(open): the open narrates itself, on a channel production cannot clamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An operator watched a production service open a 16 GB store and print nothing for three minutes before its first line of work. Two defects, both fixed here. The narration was written to `prodLog.warn`, and every environment that looks like production clamps the logger to ERROR — so the phase breakdown that would have named the slow phase was composed and thrown away. `prodLog.narrate` is always visible, like `error`: it carries the two things an operator is entitled to hear from a database regardless of a cost setting — why it is slow and what it is doing about it. `silent: true` still silences it; that is a request, not a default. And nothing spoke DURING a phase, only after the whole open. init() now runs an unref'd heartbeat that every 5s names the phase currently running, its elapsed wall and what it is paying for, plus one line per phase as it ends for any phase over 2s. The generation-log fold's own progress and completion lines move to the same channel and now carry their wall — they were invisible in production, which is how an operator came to restart a converging fold three times. Pins: tests/integration/open-narration.test.ts — narrate() survives the clamp that silences warn(); a 6.5s storage-init produces a heartbeat naming the phase and a completion line naming its wall, with the logger clamped to ERROR. --- src/brainy.ts | 70 ++++++++++++-- src/db/generationStore.ts | 13 ++- src/utils/logger.ts | 20 ++++ tests/integration/open-narration.test.ts | 114 +++++++++++++++++++++++ 4 files changed, 202 insertions(+), 15 deletions(-) create mode 100644 tests/integration/open-narration.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 957920f5..81f3cc7e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1098,21 +1098,67 @@ export class Brainy implements BrainyInterface { configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging } - // OPEN-PATH NARRATION: lightweight phase timing across the five - // named stretches of init — storage init / generation-store open+fold / - // index init+gate / VFS bootstrap / embedding-warm-started. Each - // `markPhase()` call records elapsed ms SINCE THE PREVIOUS checkpoint, - // so the buckets always sum to the pre-integration/warmOnOpen total. - // Silent under 2s; one `prodLog.warn` line naming every phase's ms - // above it, so the operator's next restart storm names its own slow - // phase instead of re-deriving it from a stack of raw timestamps. + // OPEN-PATH NARRATION: phase timing across the five named stretches of + // init — storage init / generation-store open+fold / index init+gate / + // VFS bootstrap / embedding-warm-started. Each `markPhase()` call records + // elapsed ms SINCE THE PREVIOUS checkpoint, so the buckets always sum to + // the pre-integration/warmOnOpen total. + // + // THE LAW THIS ENFORCES: an open is never silent for more than + // OPEN_HEARTBEAT_MS. A production service opening a 16 GB store logged + // NOTHING for three minutes and then began work — the operator could not + // tell a slow open from a hung one, and restarted into the same wall. + // Two mechanisms, both on the always-visible narration channel (the old + // breakdown used `prodLog.warn`, which production clamps away — that is + // why the three minutes were silent): + // - a heartbeat that names the phase currently running and its elapsed + // wall, every OPEN_HEARTBEAT_MS, for as long as the open lasts; + // - one line per phase AS IT ENDS, naming its wall and its cause, for + // any phase over OPEN_PHASE_NARRATE_MS. + // The heartbeat is unref'd and cleared in the `finally` below, so it can + // neither hold the process open nor outlive a failed init. It cannot fire + // inside a phase that blocks the event loop synchronously; such a phase + // must narrate its own progress (the generation-log fold does). + const OPEN_HEARTBEAT_MS = 5_000 + const OPEN_PHASE_NARRATE_MS = 2_000 + /** Phase order + what each one is paying for, quoted in its narration. */ + const OPEN_PHASES: ReadonlyArray<{ name: string; cause: string }> = [ + { name: 'storage-init', cause: 'opening the store and loading its count ledger' }, + { + name: 'generation-store-open-fold', + cause: 'opening the generation store: crash-recovery replay/fold, derived-family registration, format handshake' + }, + { name: 'index-init-gate', cause: 'constructing the derived indexes and gating them for serving' }, + { name: 'vfs-bootstrap', cause: 'bootstrapping the virtual filesystem' }, + { name: 'embedding-warm-started', cause: 'starting the background embedding warm' } + ] const initStart = Date.now() let lastPhaseCheckpoint = initStart + let currentPhaseIndex = 0 const phaseTimingsMs: Record = {} + const openHeartbeat: ReturnType = setInterval(() => { + const phase = OPEN_PHASES[currentPhaseIndex] + if (!phase) return + prodLog.narrate( + `[Brainy] open: still in phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` + + `"${phase.name}" after ${Math.round((Date.now() - lastPhaseCheckpoint) / 1000)}s ` + + `(${Math.round((Date.now() - initStart) / 1000)}s into the open) — ${phase.cause}` + ) + }, OPEN_HEARTBEAT_MS) + if (typeof openHeartbeat.unref === 'function') openHeartbeat.unref() const markPhase = (name: string): void => { const now = Date.now() - phaseTimingsMs[name] = now - lastPhaseCheckpoint + const elapsed = now - lastPhaseCheckpoint + phaseTimingsMs[name] = elapsed lastPhaseCheckpoint = now + const finished = OPEN_PHASES[currentPhaseIndex] + if (elapsed >= OPEN_PHASE_NARRATE_MS && finished && finished.name === name) { + prodLog.narrate( + `[Brainy] open: phase ${currentPhaseIndex + 1}/${OPEN_PHASES.length} ` + + `"${name}" finished in ${elapsed}ms — ${finished.cause}` + ) + } + currentPhaseIndex++ } try { @@ -1777,7 +1823,7 @@ export class Brainy implements BrainyInterface { const phaseList = Object.entries(phaseTimingsMs) .map(([name, ms]) => `${name}=${ms}ms`) .join(', ') - prodLog.warn( + prodLog.narrate( `[Brainy] slow open: ${totalOpenMs}ms total (${phaseList}) — see the ` + `phase breakdown above to find which one to investigate first` ) @@ -1839,6 +1885,10 @@ export class Brainy implements BrainyInterface { // log — a plain string interpolation discards both stack and cause. const message = error instanceof Error ? error.message : String(error) throw new Error(`Failed to initialize Brainy: ${message}`, { cause: error }) + } finally { + // The open is over — succeeded or failed. Stop the heartbeat here so a + // failed init never leaves a timer narrating a phase nobody is running. + clearInterval(openHeartbeat) } } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 81bed338..d925c9e0 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -652,6 +652,7 @@ export class GenerationStore { : 'WHOLE-LOG fold' : 'above-manifest replay' let replayed = 0 + const foldStartedAt = Date.now() const replayFact = async (fact: CommitFact): Promise => { for (const op of fact.ops) { let image: { metadata: unknown | null; vector: unknown | null } @@ -697,9 +698,10 @@ export class GenerationStore { } replayed++ if (replayed % 1000 === 0) { - prodLog.warn( + prodLog.narrate( `[GenerationStore] recovery fold in progress — ${replayed} fact(s) folded ` + - `(at generation ${fact.generation}); do not restart, the fold is finite` + `in ${Date.now() - foldStartedAt}ms (at generation ${fact.generation}); ` + + `do not restart, the fold is finite` ) } if (fact.generation > this.committed) { @@ -714,7 +716,7 @@ export class GenerationStore { } } if (uncleanOpen) { - prodLog.warn( + prodLog.narrate( `[GenerationStore] log-authority recovery: ${foldKind} beginning ` + `(unclean shutdown detected) — streaming replay, bounded memory, ` + `progress every 1000 facts. Do not restart the process; a restart ` + @@ -737,9 +739,10 @@ export class GenerationStore { } await this.storage.writeRawObject(MANIFEST_PATH, manifest) await this.storage.syncRawObjects([MANIFEST_PATH]) - prodLog.warn( + prodLog.narrate( `[GenerationStore] log-authority recovery replayed ${replayed} fact(s) into ` + - `canonical (${foldKind}; committed at ${this.committed}) — an acked write is never lost` + `canonical in ${Date.now() - foldStartedAt}ms (${foldKind}; committed at ` + + `${this.committed}) — an acked write is never lost` ) } // A recovery fold re-applied (and the barrier below re-syncs) every diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 5154d4fd..0d6b6594 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -266,6 +266,26 @@ export const prodLog = { console.error(message, ...args) }, + /** + * THE NARRATION CHANNEL — always visible, exactly like `error`. + * + * `warn`/`info`/`log` below are clamped to ERROR in any environment that + * looks like production (see isProductionEnvironment), which is the right + * default for chatter and the wrong one for the two things an operator is + * entitled to hear from a database no matter what: WHY IT IS SLOW and WHAT + * IT IS DOING ABOUT IT. A production service opening a 16 GB store spent + * three minutes emitting nothing at all — the phase timings that would have + * named the slow phase were written to `warn` and thrown away by the log + * level. Progress and cost narration goes here; it is never a per-record + * line, always a phase, a wall, or a bounded-cadence heartbeat. + * + * `silent: true` still silences it — that is the consumer's explicit + * request, not a cost default. + */ + narrate: (message?: any, ...args: any[]) => { + console.warn(message, ...args) + }, + // These are suppressed in production unless BRAINY_LOG_LEVEL is set warn: (message?: any, ...args: any[]) => smartConsole.warn(message, ...args), info: (message?: any, ...args: any[]) => smartConsole.info(message, ...args), diff --git a/tests/integration/open-narration.test.ts b/tests/integration/open-narration.test.ts new file mode 100644 index 00000000..95aba9f1 --- /dev/null +++ b/tests/integration/open-narration.test.ts @@ -0,0 +1,114 @@ +/** + * @module tests/integration/open-narration + * @description THE OPEN IS NEVER SILENT. + * + * A production service opened a 16 GB store and logged nothing at all for + * three minutes before its first line of work. Two defects made that possible + * and both are pinned here: + * + * 1. The phase breakdown was written to `prodLog.warn`, which every + * environment that looks like production clamps away. The narration + * channel (`prodLog.narrate`) is always visible, like `error`. + * 2. Nothing spoke DURING a phase — only after the whole open finished, if + * at all. A heartbeat now names the phase currently running and its + * elapsed wall while the open is still happening. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' +import { prodLog, configureLogger, LogLevel } from '../../src/utils/logger.js' + +function makeTempDir(): string { + return mkdtempSync(join(tmpdir(), 'brainy-open-narration-')) +} + +/** Capture console.warn lines emitted while `fn` runs. */ +async function captureWarn(fn: () => Promise): Promise<{ result: T; lines: string[] }> { + const lines: string[] = [] + const orig = console.warn + console.warn = ((...args: unknown[]) => { + lines.push(args.map((a) => String(a)).join(' ')) + }) as typeof console.warn + try { + return { result: await fn(), lines } + } finally { + console.warn = orig + } +} + +describe('open narration', () => { + let dir: string + let brain: Brainy | null = null + + beforeEach(() => { dir = makeTempDir() }) + + afterEach(async () => { + if (brain) { + try { await brain.close() } catch { /* already closed */ } + brain = null + } + try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } + }) + + it('narrate() survives the production log clamp that silences warn()', async () => { + // Exactly what isProductionEnvironment() does to the logger: level ERROR. + configureLogger({ level: LogLevel.ERROR }) + try { + const { lines } = await captureWarn(async () => { + prodLog.warn('[Brainy] this line is chatter and may be clamped') + prodLog.narrate('[Brainy] this line is why the database is slow') + }) + expect(lines.some((l) => /why the database is slow/.test(l))).toBe(true) + expect(lines.some((l) => /chatter/.test(l))).toBe(false) + } finally { + configureLogger({ level: LogLevel.INFO }) + } + }) + + it('names a slow phase as it ends, and heartbeats while it is still running', async () => { + // Seed a store, then reopen it with a deliberately slow storage init so + // the first phase crosses both the heartbeat and the narrate thresholds. + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + await brain.add({ data: 'seed entity', type: NounType.Concept }) + await brain.flush() + await brain.close() + brain = null + + const realInit = FileSystemStorage.prototype.init + FileSystemStorage.prototype.init = async function slowInit(this: FileSystemStorage) { + await new Promise((r) => setTimeout(r, 6_500)) + return realInit.call(this) + } + // Clamped to ERROR for the whole open: the narration must survive it. + configureLogger({ level: LogLevel.ERROR }) + try { + const { result, lines } = await captureWarn(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + return next + }) + brain = result + + // The heartbeat spoke DURING the phase, naming the phase and its cause. + const heartbeats = lines.filter((l) => /open: still in phase 1\/5 "storage-init"/.test(l)) + expect(heartbeats.length).toBeGreaterThanOrEqual(1) + expect(heartbeats[0]).toMatch(/loading its count ledger/) + + // And the phase named its own wall as it ended. + const ended = lines.filter((l) => /open: phase 1\/5 "storage-init" finished in \d+ms/.test(l)) + expect(ended.length).toBe(1) + + // The whole-open breakdown is on the same always-visible channel. + expect(lines.some((l) => /slow open: \d+ms total \(.*storage-init=/.test(l))).toBe(true) + } finally { + FileSystemStorage.prototype.init = realInit + configureLogger({ level: LogLevel.INFO }) + } + }, 120_000) +}) From f4e2d34b4e897274cbc33205b6e9b2779ea63be9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:28:25 -0700 Subject: [PATCH 045/150] fix(storage): a suspect count ledger heals itself, and counts.json is written atomically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a real store: the ALL-visibility ledger read 14,231 nouns against 14,056 identity records and 72,729 verbs against 72,679 — exactly that store's 25 noun and 50 verb SCAR directories. Two copies of the same archive derived different numbers (14,231 and 14,081), because each had been persisted at a different moment under the old rule that counted one entity per id DIRECTORY. A downstream index heal subtracted against that denominator and reported remaining work that did not exist. The scan already applies the right predicate — one entity per IDENTITY RECORD (the metadata content leg), shared with pruneOrphanedEntities so the two agree by construction. What was missing is that a ledger persisted under the old rule was only FLAGGED suspect and then went on serving its wrong numbers for the life of the store, waiting for an operator to run repairIndex. - The ledger now derives itself honestly in the BACKGROUND after the open, narrating start and finish with the correction it made. Background because these scalars are denominators — no read is served from them — and because walks exactly like these are how a 24,898-id store spent minutes of a restart in silence. Observable via whenCountLedgerSettled(); nothing in the read path waits on it. - A derivation that raced a write refuses to stamp its number "exact": one retry on a quiet store, then the ledger stays SUSPECT and says so, naming repairIndex as the door that recounts under a barrier. - The one derivation that CANNOT leave the foreground says why it cannot: getNounCount()/getVerbCount() are served from it, and a background walk would make a populated store answer "0 entities" — a wrong answer, not a slow one. It narrates its start and its wall instead. - counts.json is written temp+rename. A truncating write left a window — measured at roughly 750ms after a flush or close — in which a concurrent reader saw the file EMPTY; an unparseable ledger sends the next open down the full-rescan path, so the cheapest file in the store was buying the most expensive recovery. - The writer lock's clean-close record is now consulted before the same-process branch too: a restart reported "Re-acquiring writer lock ... this is a bug" immediately after a clean close, sending an operator after a leak that did not exist. Pins: tests/integration/count-ledger-identity-record.test.ts (background correction with scar and ghost fixtures, two copies of one archive agreeing, counts.json never observed unparseable across 40 persists); tests/integration/ledger-derivation-identity.test.ts updated to the new law — the OPEN still never walks (proved by slowing the walk 1.2s and timing the open), and the ledger heals behind it. --- src/storage/adapters/fileSystemStorage.ts | 253 +++++++++++++++--- .../count-ledger-identity-record.test.ts | 251 +++++++++++++++++ .../ledger-derivation-identity.test.ts | 87 ++++-- 3 files changed, 519 insertions(+), 72 deletions(-) create mode 100644 tests/integration/count-ledger-identity-record.test.ts diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index a01c2015..784ea5d8 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -120,6 +120,13 @@ export class FileSystemStorage extends BaseStorage { */ private writerHeartbeatInFlight?: Promise + /** + * The in-flight background count-ledger derivation, if one was needed at + * open. See {@link scheduleCountLedgerDerivation} — awaited only by + * {@link whenCountLedgerSettled}, never by a read. + */ + private countLedgerDerivation?: Promise + // Flush-request RPC state. The writer polls `locks/_flush_requests/` for // new `.req` files and emits `.ack` files in `locks/_flush_responses/` after // flushing. Inspectors call `requestFlushOverFilesystem` to drop a request @@ -1889,18 +1896,41 @@ export class FileSystemStorage extends BaseStorage { } } + // THE CLEAN-CLOSE RECORD IS READ BEFORE ANY VERDICT (see + // WriterCloseRecord). A lock file whose release was RECORDED is + // bookkeeping left by an orderly shutdown, not evidence of anything — + // and that is true whether the previous holder was another process or + // an earlier instance in THIS one. A production restart reported + // "Re-acquiring writer lock ... this is a bug" immediately after a clean + // close, sending an operator hunting for a leak that did not exist. + const closeRecord = existing ? await this.readWriterCloseRecord() : null + const releasedCleanly = + existing !== null && + closeRecord !== null && + this.closeRecordVouchesFor(closeRecord, existing) + if (existing) { // Same-process re-open: a second Brainy instance in this Node process // (e.g. test "simulate server restart" patterns, or a consumer that // explicitly re-instantiates without closing first). This isn't the // dangerous cross-process case the lock exists to prevent — the two // instances share a memory space and can't silently diverge from each - // other beyond what their callers already see. Warn and take over. + // other beyond what their callers already see. Warn and take over — + // unless the record proves the previous instance already let go, in + // which case there is nothing to warn about. if (existing.pid === myPid && existing.hostname === hostname && !options?.force) { - console.warn( - `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + - `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` - ) + if (releasedCleanly) { + console.warn( + `[brainy] Clearing the leftover writer lock for ${this.rootDir} — an earlier ` + + `instance in this process (PID ${existing.pid}) RELEASED it cleanly at ` + + `${closeRecord!.closedAt} but could not remove the file. Nothing to recover.` + ) + } else { + console.warn( + `[brainy] Re-acquiring writer lock for ${this.rootDir} held by the same process (PID ${existing.pid}). ` + + `If you intended to keep the previous Brainy instance alive, this is a bug — close it first.` + ) + } const info: WriterLockInfo = { pid: myPid, hostname, @@ -1915,17 +1945,11 @@ export class FileSystemStorage extends BaseStorage { return info } - // THE CLEAN-CLOSE RECORD IS CONSULTED FIRST (see WriterCloseRecord). - // A lock file whose release was RECORDED is bookkeeping left behind by - // an orderly shutdown, not evidence of a crash — take it over calmly - // and say so. Only when no record vouches for this lock do we fall - // back to inferring liveness from the pid, and then we say THAT - // honestly too: an unrecorded lock means the writer did not complete - // its close, so the store was not closed cleanly and this open pays - // recovery. - const closeRecord = await this.readWriterCloseRecord() - const releasedCleanly = - closeRecord !== null && this.closeRecordVouchesFor(closeRecord, existing) + // A cleanly-released lock is stale by RECORD, not by inference. Only + // when no record vouches for this lock do we fall back to pid + // liveness, and then we say THAT honestly too: an unrecorded lock + // means the writer did not complete its close, so the store was not + // closed cleanly and this open pays recovery. const stale = releasedCleanly || (!options?.force && (await this.isWriterLockStale(existing))) if (!options?.force && !stale) { @@ -2224,6 +2248,9 @@ export class FileSystemStorage extends BaseStorage { try { await this.writeFileAtomic(recordFile, JSON.stringify(record, null, 2)) } catch (err) { + // ENOENT = the lock directory is gone, i.e. the whole store was removed + // under us. There is no next open to inform. + if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return console.warn( `[brainy] Failed to write the writer clean-close record for ${this.rootDir} — ` + `the next open will fall back to pid liveness and may report this orderly ` + @@ -2760,25 +2787,29 @@ export class FileSystemStorage extends BaseStorage { this.allCountsDerivedBy = undefined this.allCountsSuspect = true needsPersist = true - prodLog.warn( + prodLog.narrate( '[FileSystemStorage] canonical count ledger was derived under the legacy ' + - 'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' + - 'exact denominators' + 'container rule — it counts one entity per id DIRECTORY, so every ghost/scar ' + + 'container inflates it. Marked suspect, and an honest recount is scheduled to ' + + 'run in the background after this open; until it lands, do not subtract ' + + 'against these ALL scalars.' ) + // A suspect ledger used to stay wrong for the life of the store, + // waiting for an operator to run repairIndex. A downstream index + // heal took its "remaining" figure from these inflated + // denominators and reported work that did not exist. The ledger + // now HEALS ITSELF — in the background, because a denominator is + // a derived scalar and no read is ever served from it. + this.scheduleCountLedgerDerivation('legacy container-rule ledger') } } else { - const nouns = await this.scanCanonicalEntities('nouns') - const verbs = await this.scanCanonicalEntities('verbs') - this.totalNounCountAll = nouns.count - this.totalVerbCountAll = verbs.count - this.allCountsSuspect = false - this.allCountsDerivedBy = 'identity-record' - console.warn( - `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` + - `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + - `every tier) and persisted; no further scan.` - ) - needsPersist = true + // No ALL scalars at all. There is nothing to serve in the meantime — + // a zero would read as an empty store — so the scalars stay unknown + // and SUSPECT until the background derivation lands. The open does + // not wait for it: an id-tree walk is O(ids) and this file has been + // the whole reason a 24k-id store opened in silence. + this.allCountsSuspect = true + this.scheduleCountLedgerDerivation('counts.json predates the ALL-visibility ledger') } // The vectored-noun scalar (shipped after the ALL scalars above — a @@ -2791,14 +2822,12 @@ export class FileSystemStorage extends BaseStorage { if (typeof counts.totalVectoredNounCount === 'number') { this.totalVectoredNounCount = counts.totalVectoredNounCount } else { - const vectored = await this.scanVectoredNounCount() - this.totalVectoredNounCount = vectored - console.warn( - `[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` + - `derived once by reading every noun's vectors.json (${vectored} vectored) and ` + - `persisted; no further scan.` - ) - needsPersist = true + // O(nouns) CONTENT reads — the most expensive derivation of the + // three, and the one most likely to have been the silent minutes at + // the front of a large store's open. Background, suspect until it + // lands, same as the ALL scalars. + this.allCountsSuspect = true + this.scheduleCountLedgerDerivation('counts.json predates the vectored-noun ledger') } if (needsPersist) { await this.persistCounts() @@ -2827,6 +2856,22 @@ export class FileSystemStorage extends BaseStorage { * Initialize counts by scanning disk (only done once) */ private async initializeCountsFromDisk(): Promise { + const startedAt = Date.now() + // THIS ONE CANNOT LEAVE THE FOREGROUND, and the reason is worth stating: + // it derives `totalNounCount` / `totalVerbCount`, the scalars + // `getNounCount()` and `getVerbCount()` RETURN. Backgrounding it would + // make a populated store answer "0 entities" until the walk landed — a + // wrong answer, not a slow one, and the serving law grades a failure by + // whether an answer could be wrong. The ALL-visibility denominators, which + // no read is served from, DO run in the background (see + // scheduleCountLedgerDerivation). What this walk owes the operator instead + // is narration: it announces itself, and reports its wall. + prodLog.narrate( + `[FileSystemStorage] no usable counts.json — deriving the entity counters from ` + + `the canonical id tree now. This is O(ids) listings plus one vectors.json read ` + + `per noun, and it BLOCKS the open because getNounCount()/getVerbCount() are ` + + `served from it. It runs once; the result is persisted.` + ) try { // Count the CANONICAL 8.0 layout (`entities////…`) — // the tree saveNoun/getNouns actually read and write. The previous scan @@ -2874,6 +2919,11 @@ export class FileSystemStorage extends BaseStorage { } await this.persistCounts() + prodLog.narrate( + `[FileSystemStorage] counter derivation from the canonical id tree finished in ` + + `${Date.now() - startedAt}ms: ${this.totalNounCount} nouns, ${this.totalVerbCount} verbs, ` + + `${this.totalVectoredNounCount} vectored nouns — persisted, stamped identity-record.` + ) } catch (error) { console.error('Error initializing counts from disk:', error) } @@ -2895,6 +2945,118 @@ export class FileSystemStorage extends BaseStorage { * directories (absolute paths) — nouns feed the type-distribution estimate * above. An absent tree (fresh store) counts zero. */ + /** + * @description Derive the ALL-visibility count ledger honestly — one entity + * per IDENTITY RECORD, never per id directory — IN THE BACKGROUND, once, + * and persist the result stamped `identity-record`. + * + * Why background: these scalars are DENOMINATORS. No read is served from + * them, so deriving them cannot be allowed to hold an open hostage — a + * store with 24,898 ids spent minutes of a production restart inside walks + * exactly like these, in silence, before serving anything. Why at all: a + * ledger derived under the old container rule stayed wrong for the life of + * the store, and a downstream index heal subtracted against it and reported + * remaining work that did not exist (measured on a real store: 14,231 + * derived against 14,056 identity records — precisely the store's 25 noun + * scar directories; verbs 72,729 against 72,679, its 50 verb scars). + * + * Idempotent: a second call while one is in flight joins the first. + * @param reason - What made the ledger untrustworthy, quoted in narration. + * @returns Nothing; observe completion with {@link whenCountLedgerSettled}. + */ + private scheduleCountLedgerDerivation(reason: string): void { + if (this.countLedgerDerivation) return + this.countLedgerDerivation = (async () => { + const startedAt = Date.now() + prodLog.narrate( + `[FileSystemStorage] count-ledger derivation started in the background ` + + `(${reason}) — counting identity records, not id directories; the open does ` + + `not wait for it and no read is served from these scalars.` + ) + try { + const beforeNouns = this.totalNounCountAll + const beforeVerbs = this.totalVerbCountAll + const beforeVectored = this.totalVectoredNounCount + // A walk that RACED A WRITE cannot prove its number: a row that landed + // mid-walk may or may not have been in the shard the walk had already + // passed. Rather than persist a figure that might be off by one and + // stamp it "exact", the walk is repeated once on a quiet store, and if + // the store is never quiet the ledger stays SUSPECT and says so. One + // retry, never a spin. + let attempt = 0 + let derived: { nouns: number; verbs: number; vectored: number } | null = null + while (attempt < 2 && derived === null) { + attempt++ + const activityBefore = this.ledgerActivityStamp() + const nouns = await this.scanCanonicalEntities('nouns') + const verbs = await this.scanCanonicalEntities('verbs') + const vectored = await this.scanVectoredNounCount() + if (this.ledgerActivityStamp() === activityBefore) { + derived = { nouns: nouns.count, verbs: verbs.count, vectored } + } + } + if (derived === null) { + this.allCountsSuspect = true + prodLog.narrate( + `[FileSystemStorage] count-ledger derivation could not finish on a quiet store ` + + `after ${attempt} attempts (${Date.now() - startedAt}ms) — writes landed during ` + + `every walk. The ALL-visibility scalars stay SUSPECT and must not be subtracted ` + + `against; brain.repairIndex() derives them under a recount barrier.` + ) + return + } + this.totalNounCountAll = derived.nouns + this.totalVerbCountAll = derived.verbs + this.totalVectoredNounCount = derived.vectored + this.allCountsDerivedBy = 'identity-record' + this.allCountsSuspect = false + await this.persistCounts() + prodLog.narrate( + `[FileSystemStorage] count-ledger derivation finished in ${Date.now() - startedAt}ms: ` + + `${derived.nouns} nouns / ${derived.verbs} verbs / ${derived.vectored} vectored nouns` + + (beforeNouns !== derived.nouns || + beforeVerbs !== derived.verbs || + beforeVectored !== derived.vectored + ? ` (corrected from ${beforeNouns} / ${beforeVerbs} / ${beforeVectored} — the ` + + `difference is ghost and scar containers the old rule counted as entities)` + : ' (unchanged)') + + ` — persisted, stamped identity-record, no longer suspect.` + ) + } catch (error) { + // The ledger stays suspect and the next open retries. Loud: a + // denominator nobody can derive is a fact an operator must have. + this.allCountsSuspect = true + prodLog.error( + `[FileSystemStorage] count-ledger derivation FAILED after ` + + `${Date.now() - startedAt}ms — the ALL-visibility scalars remain SUSPECT ` + + `and must not be subtracted against; the next open retries:`, + error + ) + } + })() + } + + /** + * @description A cheap witness that the ledger changed while a walk was + * running. Every landed write moves one of these live counters, so an + * unchanged stamp across a walk means no write landed during it. + * @returns A value that differs whenever the live ALL scalars have moved. + */ + private ledgerActivityStamp(): string { + return `${this.totalNounCountAll}:${this.totalVerbCountAll}:${this.totalVectoredNounCount}` + } + + /** + * @description Resolve once any background count-ledger derivation has + * settled (succeeded or failed). Resolves immediately when none was needed. + * Exists so tests and operators can observe the ledger's honest value rather + * than race it; nothing in the read path waits on this. + * @returns A promise that settles with the derivation. + */ + public async whenCountLedgerSettled(): Promise { + await this.countLedgerDerivation + } + private async scanCanonicalEntities( kind: 'nouns' | 'verbs' ): Promise<{ count: number; sampleDirs: string[] }> { @@ -3053,10 +3215,15 @@ export class FileSystemStorage extends BaseStorage { lastUpdated: new Date().toISOString() } - await fs.promises.writeFile( - this.countsFilePath, - JSON.stringify(counts, null, 2) - ) + // ATOMIC (temp + rename), never a plain writeFile. A direct write + // truncates the file first, so every persist opened a window — measured + // at roughly 750ms after a flush or close on a real store — in which a + // concurrent reader saw counts.json EMPTY. An empty file is unparseable, + // and an unparseable ledger sends the next open down the full-rescan + // path: the cheapest file in the store was costing the most expensive + // recovery. The rename is atomic, so a reader sees the old ledger or the + // new one, never neither. + await this.writeFileAtomic(this.countsFilePath, JSON.stringify(counts, null, 2)) } catch (error) { console.error('Error persisting counts:', error) } diff --git a/tests/integration/count-ledger-identity-record.test.ts b/tests/integration/count-ledger-identity-record.test.ts new file mode 100644 index 00000000..1066213a --- /dev/null +++ b/tests/integration/count-ledger-identity-record.test.ts @@ -0,0 +1,251 @@ +/** + * @module tests/integration/count-ledger-identity-record + * @description THE COUNT LEDGER COUNTS RECORDS, NOT DIRECTORIES — and heals + * itself when it was derived the other way. + * + * Measured on a real store: the ALL-visibility ledger read 14,231 nouns + * against 14,056 identity records, and 72,729 verbs against 72,679 — exactly + * that store's 25 noun and 50 verb SCAR directories (empty `/` containers + * left by a pre-8.3.1 partial delete). Two copies of the SAME archive derived + * different numbers, because each had been persisted at a different moment + * under the old container rule. A downstream index heal subtracted against + * those denominators and reported remaining work that did not exist. + * + * The membership predicate is the IDENTITY RECORD (the metadata content leg). + * The scan already applies it; what is pinned here is that a ledger persisted + * under the OLD rule does not go on lying — it is corrected in the background, + * without blocking the open, and two copies of one archive agree. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { + mkdtempSync, + mkdirSync, + rmSync, + writeFileSync, + readFileSync, + cpSync, + existsSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { FileSystemStorage as FileSystemStorageClass } from '../../src/storage/adapters/fileSystemStorage.js' +import type { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' + +const NOUN_COUNT = 6 +const NOUN_SCARS = 3 +const VERB_SCARS = 2 +/** A REAL two-hex shard — the scan skips any directory that is not one. */ +const SCAR_SHARD = 'ab' + +function makeTempDir(): string { + return mkdtempSync(join(tmpdir(), 'brainy-count-ledger-')) +} + +/** The FileSystemStorage behind a brain. */ +function storageOf(brain: Brainy): FileSystemStorage { + return (brain as unknown as { storage: FileSystemStorage }).storage +} + +/** + * Add `count` empty `/` container directories under + * `entities///` — scars, exactly as a partial delete leaves them. + */ +function addScarContainers(dir: string, kind: 'nouns' | 'verbs', count: number): void { + for (let i = 0; i < count; i++) { + const id = `${SCAR_SHARD}5ca4000-0000-0000-0000-00000000000${i}` + mkdirSync(join(dir, 'entities', kind, SCAR_SHARD, id), { recursive: true }) + } +} + +/** Add one GHOST container: a `vectors.json` leg with no identity record. */ +function addGhostContainer(dir: string): void { + const id = `${SCAR_SHARD}9405700-0000-0000-0000-000000000000` + const idDir = join(dir, 'entities', 'nouns', SCAR_SHARD, id) + mkdirSync(idDir, { recursive: true }) + writeFileSync(join(idDir, 'vectors.json'), JSON.stringify({ id, vector: [0.1, 0.2] })) +} + +/** + * Rewrite counts.json into the LEGACY shape: ALL scalars inflated by the + * containers, and no `allCountsDerivedBy` stamp — exactly what a store carried + * when it was last written by a build that counted directories. + */ +function writeLegacyCountsLedger(dir: string, inflateNouns: number, inflateVerbs: number): void { + const file = join(dir, '_system', 'counts.json') + const counts = JSON.parse(readFileSync(file, 'utf-8')) + counts.totalNounCountAll = (counts.totalNounCountAll ?? 0) + inflateNouns + counts.totalVerbCountAll = (counts.totalVerbCountAll ?? 0) + inflateVerbs + delete counts.allCountsDerivedBy + delete counts.allCountsSuspect + writeFileSync(file, JSON.stringify(counts, null, 2)) +} + +/** + * Seed a store and return the HONEST ledger it holds when freshly written — + * the baseline the correction must return to. Read from the engine rather than + * hardcoded: an open creates its own rows (the VFS root), and a pin that + * asserts a literal would be pinning that incidental fact instead of the rule. + */ +async function seedStore(dir: string): Promise<{ nouns: number; verbs: number }> { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + const ids: string[] = [] + for (let i = 0; i < NOUN_COUNT; i++) { + ids.push(await brain.add({ data: `entity number ${i}`, type: NounType.Concept })) + } + await brain.relate({ from: ids[0], to: ids[1], type: 'relatedTo' } as never) + await brain.relate({ from: ids[1], to: ids[2], type: 'relatedTo' } as never) + await brain.flush() + const ledger = await storageOf(brain).getCanonicalCounts() + const baseline = { nouns: ledger.nouns.all, verbs: ledger.verbs.all } + await brain.close() + return baseline +} + +/** + * Make the ledger walk take `ms` so a test can observe the open completing + * WITHOUT it. Patches the prototype before any brain is constructed; returns + * the restore function. + */ +function slowTheLedgerWalk(ms: number): () => void { + const proto = ( + FileSystemStorageClass as unknown as { + prototype: Record Promise> + } + ).prototype + const real = proto.scanCanonicalEntities + proto.scanCanonicalEntities = async function slow(this: unknown, ...args: unknown[]) { + await new Promise((r) => setTimeout(r, ms)) + return real.apply(this, args) + } + return () => { proto.scanCanonicalEntities = real } +} + +describe('the canonical count ledger', () => { + const dirs: string[] = [] + + afterEach(() => { + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + function trackDir(): string { + const dir = makeTempDir() + dirs.push(dir) + return dir + } + + it('corrects a legacy container-rule ledger in the background, counting identity records', async () => { + const dir = trackDir() + const baseline = await seedStore(dir) + + // Scars and a ghost: containers with no identity record. + addScarContainers(dir, 'nouns', NOUN_SCARS) + addScarContainers(dir, 'verbs', VERB_SCARS) + addGhostContainer(dir) + // The ledger as the old rule left it: every container counted. + writeLegacyCountsLedger(dir, NOUN_SCARS + 1, VERB_SCARS) + + const restore = slowTheLedgerWalk(1_500) + let brain: Brainy + try { + const openStarted = Date.now() + brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + const openMs = Date.now() - openStarted + const storage = storageOf(brain) + + // THE OPEN DID NOT WAIT. Two walks of 1.5s each would have added 3s. + expect(openMs).toBeLessThan(2_500) + // And while it runs, the scalars say so instead of being subtracted against. + const atOpen = await storage.getCanonicalCounts() + expect(atOpen.suspect).toBe(true) + expect(atOpen.nouns.all).toBe(baseline.nouns + NOUN_SCARS + 1) + + await storage.whenCountLedgerSettled() + } finally { + restore() + } + const storage = storageOf(brain!) + + const healed = await storage.getCanonicalCounts() + expect(healed.nouns.all).toBe(baseline.nouns) + expect(healed.verbs.all).toBe(baseline.verbs) + expect(healed.suspect).toBe(false) + + // And it is PERSISTED with the honest stamp — the correction survives a + // reopen instead of being re-derived (or re-lost) every time. + await brain!.close() + const persisted = JSON.parse(readFileSync(join(dir, '_system', 'counts.json'), 'utf-8')) + expect(persisted.totalNounCountAll).toBe(baseline.nouns) + expect(persisted.totalVerbCountAll).toBe(baseline.verbs) + expect(persisted.allCountsDerivedBy).toBe('identity-record') + + const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await reopened.init() + const afterReopen = await storageOf(reopened).getCanonicalCounts() + expect(afterReopen.nouns.all).toBe(baseline.nouns) + expect(afterReopen.suspect).toBe(false) + await reopened.close() + }, 180_000) + + it('derives the same number from two copies of one archive', async () => { + const source = trackDir() + const baseline = await seedStore(source) + addScarContainers(source, 'nouns', NOUN_SCARS) + addGhostContainer(source) + + // Two copies of the SAME bytes, each carrying a DIFFERENT legacy ledger — + // the situation that made one archive report 14,231 and its twin 14,081. + const copyA = trackDir() + const copyB = trackDir() + cpSync(source, copyA, { recursive: true }) + cpSync(source, copyB, { recursive: true }) + writeLegacyCountsLedger(copyA, NOUN_SCARS + 1, 0) + writeLegacyCountsLedger(copyB, 1, 0) + + const derived: number[] = [] + for (const dir of [copyA, copyB]) { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + const storage = storageOf(brain) + await storage.whenCountLedgerSettled() + derived.push((await storage.getCanonicalCounts()).nouns.all) + await brain.close() + } + expect(derived[0]).toBe(derived[1]) + expect(derived[0]).toBe(baseline.nouns) + }, 180_000) + + it('writes counts.json atomically — no reader ever sees it empty', async () => { + const dir = trackDir() + await seedStore(dir) + const file = join(dir, '_system', 'counts.json') + expect(existsSync(file)).toBe(true) + + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + const storage = storageOf(brain) + + // Watch the ledger across many persists. A truncating write leaves a + // window in which the file parses as nothing; a temp+rename never does. + let sawUnparseable = 0 + const watcher = setInterval(() => { + try { + JSON.parse(readFileSync(file, 'utf-8')) + } catch { + sawUnparseable++ + } + }, 1) + for (let i = 0; i < 40; i++) { + await (storage as unknown as { persistCounts: () => Promise }).persistCounts() + } + clearInterval(watcher) + await brain.close() + expect(sawUnparseable).toBe(0) + }, 180_000) +}) diff --git a/tests/integration/ledger-derivation-identity.test.ts b/tests/integration/ledger-derivation-identity.test.ts index cb19af4a..7d09e893 100644 --- a/tests/integration/ledger-derivation-identity.test.ts +++ b/tests/integration/ledger-derivation-identity.test.ts @@ -11,14 +11,22 @@ * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per * metadata content leg (`metadata.json` or `.json.gz`), the same test * `pruneOrphanedEntities()` uses, so the two agree by construction. - * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that - * carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'` - * stamp predates this fix; loading it marks `suspect = true` from a - * single field read alone, never a directory walk, and warns exactly - * once naming the cause. - * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned - * containers, recounts from the canonical metadata.json walk, and - * re-stamps — suspect clears and the ALL scalar is exact again. + * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AND THE OPEN NEVER WALKS — a + * counts.json that carries the ALL scalars but no + * `allCountsDerivedBy: 'identity-record'` stamp predates this fix; + * loading it marks `suspect = true` from a single field read alone and + * warns exactly once naming the cause. The open itself never pays a + * directory walk. + * (2b) AND IT HEALS ITSELF. The ledger used to stay wrong for the life of the + * store, waiting for an operator to run `repairIndex()` — and a + * downstream index heal subtracted against the inflated denominator and + * reported work that did not exist. An honest derivation now runs in the + * BACKGROUND after the open (never blocking it, observable via + * `whenCountLedgerSettled()`), and refuses to stamp a number it derived + * while writes were landing. + * (3) THE SANCTIONED RECOUNT ALSO CLEARS IT — `repairIndex()` prunes the + * orphaned containers, recounts from the canonical metadata.json walk, + * and re-stamps — the ALL scalar is exact and the containers are gone. * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store * with no counts.json stamps as it writes, so a brand-new store never * carries the legacy signature. @@ -115,29 +123,43 @@ describe('ledger derivation identity — the ALL scalar is the identity-record p delete raw.allCountsDerivedBy fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) - const warnSpy = vi.spyOn(prodLog, 'warn') - // The two derivation walks live on FileSystemStorage's prototype — - // spying here (rather than on fs.promises.readdir globally) isolates - // THIS code path's behavior from unrelated walks elsewhere in the open - // sequence (a separate, pre-existing engine's own O(store) cost — not - // this fix's concern, and not something this pin should be sensitive - // to). Neither derivation method may run: the stamp check is a field - // read on the already-parsed counts.json, nothing more. - const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities') - const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount') + const narrateSpy = vi.spyOn(prodLog, 'narrate') + // The derivation walks live on FileSystemStorage's prototype. Slow them + // deliberately: the OPEN must not wait for them, and on a two-row store a + // real walk finishes too fast to tell "not awaited" from "instant". + const proto = FileSystemStorage.prototype as any + const realScanEntities = proto.scanCanonicalEntities + let scanEntitiesCalls = 0 + proto.scanCanonicalEntities = async function slow(this: any, ...args: any[]) { + scanEntitiesCalls++ + await new Promise((r) => setTimeout(r, 1_200)) + return realScanEntities.apply(this, args) + } + try { + const openStarted = Date.now() + brain = await open() + const openMs = Date.now() - openStarted - brain = await open() + // THE OPEN DID NOT WALK: two slowed walks would have added 2.4s to it. + expect(openMs).toBeLessThan(2_000) - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(true) + // The stamp check itself is an O(1) field read, and it names the cause. + const atOpen = await brain.storage.getCanonicalCounts() + expect(atOpen.suspect).toBe(true) + const stampWarnings = narrateSpy.mock.calls.filter( + ([msg]: any[]) => String(msg).includes('legacy') && String(msg).includes('container rule') + ) + expect(stampWarnings.length).toBe(1) // exactly one, loud - const stampWarnings = warnSpy.mock.calls.filter( - ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule') - ) - expect(stampWarnings.length).toBe(1) // exactly one, loud - - expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk - expect(scanVectoredSpy).not.toHaveBeenCalled() + // ...and the honest derivation is already running behind the open. + await brain.storage.whenCountLedgerSettled() + expect(scanEntitiesCalls).toBeGreaterThan(0) + const healed = await brain.storage.getCanonicalCounts() + expect(healed.suspect).toBe(false) + expect(healed.nouns.all).toBe(raw.totalNounCountAll) + } finally { + proto.scanCanonicalEntities = realScanEntities + } await brain.close() }) @@ -163,7 +185,14 @@ describe('ledger derivation identity — the ALL scalar is the identity-record p fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) brain = await open() - expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load + // Named suspect at load, then healed in the background WITHOUT the + // operator asking — the inflated container count is corrected to the + // identity-record population, though the orphaned containers themselves + // are still on disk (only repairIndex() removes those). + await brain.storage.whenCountLedgerSettled() + let healed = await brain.storage.getCanonicalCounts() + expect(healed.suspect).toBe(false) + expect(healed.nouns.all).toBe(realTotal) await brain.repairIndex() From 3fffd9c6e66f6e67b1eae203f470da3312427d00 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:31:42 -0700 Subject: [PATCH 046/150] feat(repair): repairIndex narrates every phase and its receipt carries the walls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a production store (14,647 nouns / 73,070 verbs) a repairIndex() ran for more than thirty minutes at roughly a full core with ZERO log lines between its start and its end, while the read doors kept serving. The operator could tell it was alive only from `top`, and could not tell which of its single-threaded walks it was inside. Same law as the open, applied to the repair: - every phase announces itself BEFORE it works, naming what it is about to walk (each canonical walk, the VFS containment reconciliation, each provider's invariant pass); - an unref'd heartbeat names the phase still running every 5s, for as long as it runs; - every phase reports its own wall, and that wall is carried in the TYPED receipt as RepairFamilyReport.durationMs — a receipt that cannot say where the time went is not a receipt; - the whole repair's narration moves to the always-visible channel, so a production log level cannot silence it. The phases move into runRepairIndexPhases() so the heartbeat can live in a finally around them; the public door and its report shape are unchanged apart from the added durationMs. Pins: tests/integration/repair-narration.test.ts — every checked family has a start line, a finish line with its wall, and a numeric durationMs in the receipt; a phase slowed to 6.5s produces a heartbeat naming it, with the logger clamped to ERROR. --- src/brainy.ts | 128 ++++++++++++++++++--- src/types/brainy.types.ts | 7 ++ tests/integration/repair-narration.test.ts | 119 +++++++++++++++++++ 3 files changed, 240 insertions(+), 14 deletions(-) create mode 100644 tests/integration/repair-narration.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 81f3cc7e..24407018 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -18199,17 +18199,89 @@ export class Brainy implements BrainyInterface { * invariant-driven pass (it was already rebuilt unconditionally — a second, * report-driven pass over the same family would be redundant at best). * + * NARRATION IS PART OF THE CONTRACT. A repair on a production store ran for + * more than thirty minutes at a full core with NOT ONE log line between its + * start and its end while the doors kept serving; the operator could tell it + * was alive only from `top`. Every phase now announces itself before it + * works, a heartbeat names the phase still running every five seconds, and + * each phase reports its own wall — carried in the receipt as + * `durationMs` per family, so nobody has to infer progress from CPU. + * * @param options.rebuild - Family name(s) to unconditionally rebuild, or `'all'` for all three (`'metadata' | 'graph' | 'vector'`). - * @returns The full per-family receipt (see {@link RepairReport}); also narrated via `prodLog.warn`. + * @returns The full per-family receipt (see {@link RepairReport}); also narrated as it goes. */ async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { await this.ensureInitialized() const startedAt = Date.now() const families: RepairFamilyReport[] = [] - const record = (family: string, entry: Omit): void => { - families.push({ family, ...entry }) + + // THE REPAIR HEARTBEAT — the same law the open obeys: no stretch of work + // may be silent for more than REPAIR_HEARTBEAT_MS. Unref'd (it never holds + // a process open) and cleared in the `finally` below. + const REPAIR_HEARTBEAT_MS = 5_000 + let currentPhase = 'starting' + let currentPhaseCause = 'preparing the repair' + let phaseStartedAt = Date.now() + const heartbeat = setInterval(() => { + prodLog.narrate( + `[Brainy] repairIndex: still in "${currentPhase}" after ` + + `${Math.round((Date.now() - phaseStartedAt) / 1000)}s ` + + `(${Math.round((Date.now() - startedAt) / 1000)}s into the repair) — ${currentPhaseCause}` + ) + }, REPAIR_HEARTBEAT_MS) + if (typeof heartbeat.unref === 'function') heartbeat.unref() + + /** Announce a phase before it does any work, and start its clock. */ + const beginPhase = (name: string, cause: string): void => { + currentPhase = name + currentPhaseCause = cause + phaseStartedAt = Date.now() + prodLog.narrate(`[Brainy] repairIndex: "${name}" started — ${cause}`) } + /** + * Close the current phase: stamp its wall into the receipt row and say + * what it did. Every family row carries its own `durationMs`. + */ + const record = (family: string, entry: Omit): void => { + const durationMs = Date.now() - phaseStartedAt + families.push({ family, ...entry, durationMs }) + prodLog.narrate( + `[Brainy] repairIndex: "${family}" finished in ${durationMs}ms — ` + + (entry.checked + ? `${entry.healed} heal(s)${entry.rebuilt ? ', rebuilt' : ''}` + + (entry.detail ? ` (${entry.detail})` : '') + : `skipped (${entry.skipped ?? entry.reason ?? 'no reason given'})`) + ) + phaseStartedAt = Date.now() + } + + try { + return await this.runRepairIndexPhases(options, families, record, beginPhase, startedAt) + } finally { + clearInterval(heartbeat) + } + } + + /** + * @description The phases of {@link repairIndex}, separated so its heartbeat + * can live in a `finally` around them. Not a public door — see `repairIndex` + * for the contract. + * @param options - As `repairIndex`. + * @param families - The receipt rows being accumulated. + * @param record - Closes a phase: stamps its wall and narrates its outcome. + * @param beginPhase - Announces a phase before it works. + * @param startedAt - When the repair began, for the closing line. + * @returns The full receipt. + */ + private async runRepairIndexPhases( + options: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' } | undefined, + families: RepairFamilyReport[], + record: (family: string, entry: Omit) => void, + beginPhase: (name: string, cause: string) => void, + startedAt: number + ): Promise { + // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete // defect: a delete that removed the metadata (content) leg but left the // vector leg + the entity directory (a "ghost"), or left an empty directory @@ -18223,6 +18295,10 @@ export class Brainy implements BrainyInterface { rebuildSubtypeCounts?: () => Promise } if (typeof pruner.pruneOrphanedEntities === 'function') { + beginPhase( + 'orphaned-containers', + 'walking every canonical id directory for ghost/scar containers left by a partial delete' + ) const orphans = await pruner.pruneOrphanedEntities() const pruned = orphans.nouns.length + orphans.verbs.length record('orphaned-containers', { @@ -18233,7 +18309,7 @@ export class Brainy implements BrainyInterface { : {}) }) if (pruned > 0) { - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` + `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` + `partial delete.` @@ -18249,6 +18325,10 @@ export class Brainy implements BrainyInterface { // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup // (scalar totals + per-type maps + type-statistics arrays) from one // canonical walk and persists them. + beginPhase( + 'count-rollups', + 'ONE canonical walk recomputing every counter rollup — scalar totals, per-type maps, type statistics' + ) await pruner.rebuildTypeCounts?.() await pruner.rebuildSubtypeCounts?.() record('count-rollups', { @@ -18271,6 +18351,10 @@ export class Brainy implements BrainyInterface { // concurrent writers. Canonical metadata.path is the truth; only VFS // containment edges are touched. Loud per repair. if (this._vfsInitialized && this._vfs) { + beginPhase( + 'vfs-containment', + 'reconciling VFS containment edges against canonical metadata.path' + ) const containment = await this._vfs.repairContainment() record('vfs-containment', { checked: true, @@ -18280,7 +18364,7 @@ export class Brainy implements BrainyInterface { : {}) }) if (containment.removed + containment.restored > 0) { - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` + `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).` ) @@ -18291,17 +18375,25 @@ export class Brainy implements BrainyInterface { record('vfs-containment', { checked: false, healed: 0, skipped: 'VFS not initialized' }) } + beginPhase( + 'metadata-corruption', + 'detect-and-repair pass over the metadata index' + ) await this.metadataIndex.detectAndRepairCorruption() record('metadata-corruption', { checked: true, healed: 0, detail: 'detect-and-repair pass ran (see its own narration for repairs)' }) // Lift a failed-rollback write-quarantine: force a full rebuild so the // derived indexes are provably reconciled with canonical, then clear the // flag so writes resume. if (this.storeInconsistency) { + beginPhase( + 'write-quarantine', + 'full derived-index rebuild to lift the quarantine set by a failed transaction rollback' + ) await this.rebuildIndexesIfNeeded(true) const cleared = this.storeInconsistency record('write-quarantine', { checked: true, healed: 1, detail: `lifted (${cleared.records.length} record(s) reconciled)` }) this.storeInconsistency = null - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` + `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` + `Writes are re-enabled.` @@ -18332,9 +18424,9 @@ export class Brainy implements BrainyInterface { record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no rebuild() contract' }) continue } - prodLog.warn( - `[Brainy] repairIndex(): explicit rebuild requested for '${familyName}' — ` + - `rebuilding unconditionally (no invariant consulted).` + beginPhase( + `provider:${familyName}`, + `explicit rebuild requested — rebuilding '${familyName}' unconditionally, no invariant consulted` ) // The metadata family routes through the online build-beside // orchestrator (B3 D3) instead of the provider's own rebuild() — @@ -18351,7 +18443,7 @@ export class Brainy implements BrainyInterface { rebuilt: true, reason: 'explicit rebuild requested' }) - prodLog.warn(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) + prodLog.narrate(`[Brainy] repairIndex(): '${familyName}' rebuild complete.`) continue } @@ -18360,11 +18452,16 @@ export class Brainy implements BrainyInterface { rebuild?: () => Promise } | null if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') { + beginPhase(`provider:${familyName}`, 'checking the provider contract') record(`provider:${familyName}`, { checked: false, healed: 0, skipped: 'no validateInvariants/rebuild contract' }) continue } + beginPhase( + `provider:${familyName}`, + `reading the '${familyName}' provider's own invariant report, then healing only what it asks for` + ) let report: ProviderInvariantReport try { report = await p.validateInvariants() @@ -18381,7 +18478,7 @@ export class Brainy implements BrainyInterface { checked: true, healed: 1, detail: `rebuilt from canonical (failing: ${report.invariants.filter((i) => !i.holds).map((i) => i.name).join(', ')})` }) - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + `requiring a rebuild — reconciling its derived state from canonical.` ) @@ -18405,7 +18502,7 @@ export class Brainy implements BrainyInterface { const failingRepairs = report.invariants .filter((i) => !i.holds && i.heal === 'repair') .map((i) => i.name) - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex(): provider '${report.provider}' asks for an incremental ` + `repair (${failingRepairs.join(', ')}) — running its own repair().` ) @@ -18444,6 +18541,7 @@ export class Brainy implements BrainyInterface { // rebuild failure are now reconciled — clear the queryable degraded state // and re-arm the read-path warning. if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) { + beginPhase('degraded-read-state', 'clearing degraded ids and re-arming the read-path warning') this._indexDegradedIds.clear() this._indexRebuildFailed = null this._degradedReadWarned = false @@ -18452,11 +18550,13 @@ export class Brainy implements BrainyInterface { const healedTotal = families.reduce((n, f) => n + f.healed, 0) const report: RepairReport = { families, healedTotal, durationMs: Date.now() - startedAt } - prodLog.warn( + prodLog.narrate( `[Brainy] repairIndex complete in ${report.durationMs}ms — ` + `${families.filter((f) => f.checked).length}/${families.length} families checked, ` + `${healedTotal} heal(s): ` + - families.map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}`).join(', ') + families + .map((f) => `${f.family}=${f.checked ? f.healed : 'skipped'}@${f.durationMs ?? 0}ms`) + .join(', ') ) return report } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index d9934c3b..a0d55c1e 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -1217,6 +1217,13 @@ export interface RepairFamilyReport { skipped?: string /** Why the outcome is what it is when neither `detail` nor `skipped` says it. */ reason?: string + /** + * The phase's own wall, in milliseconds. A repair on a production store ran + * for over thirty minutes without a single line of output; an operator had + * to read `top` to know it was alive. A receipt that cannot say WHERE the + * time went is not a receipt — every row carries its own. + */ + durationMs?: number } /** The full receipt returned by repairIndex(). */ diff --git a/tests/integration/repair-narration.test.ts b/tests/integration/repair-narration.test.ts new file mode 100644 index 00000000..1fbe15e4 --- /dev/null +++ b/tests/integration/repair-narration.test.ts @@ -0,0 +1,119 @@ +/** + * @module tests/integration/repair-narration + * @description A REPAIR NARRATES ITSELF, AND ITS RECEIPT SAYS WHERE THE TIME + * WENT. + * + * On a production store (14,647 nouns / 73,070 verbs) a `repairIndex()` ran + * for more than thirty minutes at roughly a full core with ZERO log lines + * between its start and its end, while the read doors kept serving. The + * operator could tell it was alive only from `top`, and could not tell which + * of its single-threaded walks it was inside. The law pinned here: + * + * - every phase announces itself BEFORE it works, naming what it is about + * to walk; + * - a heartbeat names the phase still running, at a bounded cadence, for as + * long as it runs; + * - every phase reports its own wall, and that wall is carried in the typed + * receipt (`RepairFamilyReport.durationMs`) — not only in a log line. + * + * All of it on the narration channel, which production's log clamp cannot + * silence (see tests/integration/open-narration.test.ts). + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' +import { prodLog, configureLogger, LogLevel } from '../../src/utils/logger.js' + +describe('repairIndex narration', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + configureLogger({ level: LogLevel.INFO }) + }) + + async function seededBrain(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-repair-narration-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + for (let i = 0; i < 5; i++) { + await brain.add({ data: `repair subject ${i}`, type: NounType.Concept }) + } + await brain.flush() + return brain + } + + it('announces every phase, reports its wall, and carries that wall in the receipt', async () => { + const brain = await seededBrain() + const narrateSpy = vi.spyOn(prodLog, 'narrate') + + const report = await brain.repairIndex() + + const lines = narrateSpy.mock.calls.map(([m]) => String(m)) + + // Every family that ran has BOTH a start line and a finish line naming it. + for (const family of report.families) { + const started = lines.filter((l) => l.includes(`"${family.family}" started —`)) + const finished = lines.filter((l) => + new RegExp(`"${family.family}" finished in \\d+ms`).test(l) + ) + expect(finished.length, `no finish line for ${family.family}`).toBeGreaterThanOrEqual(1) + // A skipped family may be recorded without a start line only if it never + // began; every family that began must have announced itself. + if (family.checked) { + expect(started.length, `no start line for ${family.family}`).toBeGreaterThanOrEqual(1) + } + // THE RECEIPT CARRIES THE WALL — not only the log. + expect(typeof family.durationMs, `${family.family} has no durationMs`).toBe('number') + expect(family.durationMs).toBeGreaterThanOrEqual(0) + } + + // The closing line accounts for the whole repair, per family. + const closing = lines.filter((l) => /repairIndex complete in \d+ms/.test(l)) + expect(closing.length).toBe(1) + expect(closing[0]).toMatch(/@\d+ms/) + }, 180_000) + + it('heartbeats while a single phase is still walking', async () => { + const brain = await seededBrain() + + // Make one phase long enough to cross the heartbeat cadence, exactly as a + // multi-minute canonical walk does on a real store. + const proto = FileSystemStorage.prototype as unknown as Record< + string, + (...args: unknown[]) => Promise + > + const realPrune = proto.pruneOrphanedEntities + proto.pruneOrphanedEntities = async function slow(this: unknown, ...args: unknown[]) { + await new Promise((r) => setTimeout(r, 6_500)) + return realPrune.apply(this, args) + } + // Clamped as production clamps it: the narration must survive. + configureLogger({ level: LogLevel.ERROR }) + const narrateSpy = vi.spyOn(prodLog, 'narrate') + try { + await brain.repairIndex() + } finally { + proto.pruneOrphanedEntities = realPrune + } + + const beats = narrateSpy.mock.calls + .map(([m]) => String(m)) + .filter((l) => /repairIndex: still in "orphaned-containers" after \d+s/.test(l)) + expect(beats.length).toBeGreaterThanOrEqual(1) + expect(beats[0]).toMatch(/ghost\/scar containers/) + }, 180_000) +}) From f5a6cb3f618611a23a5559fbada69bb41b907bb1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:44:38 -0700 Subject: [PATCH 047/150] =?UTF-8?q?perf(flush):=20an=20idle=20brain=20does?= =?UTF-8?q?=20no=20work=20=E2=80=94=20no=20periodic=20flush=20without=20a?= =?UTF-8?q?=20write?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REPORTED from the field: a process holding many stores, with no writes for ten minutes, printed "All indexes flushed to disk in 216-601ms" per store every ~35 seconds and burned over a core at idle. Every one of those flushes re-persisted state identical to what was already on disk — the provider flushes, the watermark stamps, the generation counter, the entity-tree stamp — because flush() never asked whether anything had changed. - flush() over a clean brain is now O(1) and silent: a dirty witness is set by every committed write (both commit paths end at noteWriteForPersistence, and the deferred-embed worker lands through the single-op path) and cleared by a flush that runs. A write landing DURING a flush sets it again, so no write's work is ever skipped — it is done by the next flush. Set before the policy check, so a `'manual'` consumer's explicit flush is never a no-op it didn't ask for. - An explicit flush now tells the cadence it happened. It didn't, so the very next write saw "30s since the last flush" and kicked a background flush with nothing to do, and the idle timer fired two seconds later over writes the explicit flush had already persisted. - The graph adjacency index's auto-flush asks before it acts: two O(1) reads of the LSM MemTables, and a tick over a quiet index returns without calling into the trees at all. assessProviderHealth is NOT timer-driven — it is a synchronous O(1) read of a provider's own healthReport(), called on the read gate, so it costs nothing on an idle brain. No change needed there. Pins: tests/integration/idle-costs-nothing.test.ts — 90 idle seconds produce zero flushes, zero provider calls and zero log lines; three explicit flushes over a clean brain call no provider; one write earns exactly one flush. --- src/brainy.ts | 39 +++++ src/graph/graphAdjacencyIndex.ts | 11 ++ src/graph/lsm/LSMTree.ts | 11 ++ tests/integration/idle-costs-nothing.test.ts | 147 +++++++++++++++++++ 4 files changed, 208 insertions(+) create mode 100644 tests/integration/idle-costs-nothing.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 24407018..013885ff 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -740,6 +740,18 @@ export class Brainy implements BrainyInterface { // Write acks NEVER await it; a failed background flush is LOUD and re-armed. private _persistDirtyWrites = 0 private _persistLastFlushAt = Date.now() + /** + * Whether a write has been committed since the last flush that ran. THE + * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written + * to has nothing to make durable, and a flush over it must cost nothing and + * say nothing. Measured on a production process holding 21 brains: with no + * writes for ten minutes it still printed "All indexes flushed to disk in + * 216–601ms" per brain every ~35s and idled at 1.26 cores, because a flush + * called every provider, stamped the watermarks, persisted the generation + * counter and re-stamped the entity tree whether or not anything had + * changed. + */ + private _dirtySinceLastFlush = false private _persistIdleTimer: ReturnType | null = null private _persistBackgroundFlight: Promise | null = null @@ -2668,6 +2680,12 @@ export class Brainy implements BrainyInterface { * engine's own cadence (callers never call flush() in hot paths). */ private noteWriteForPersistence(): void { + // THE DIRTY WITNESS. Set on every committed write — both commit paths + // (single-op and transaction) end here, and the deferred-embed worker + // lands its vectors through the single-op path — BEFORE the policy check, + // so a `'manual'` consumer's explicit flush() is never skipped either. + // Cleared by a flush that actually runs; see flush(). + this._dirtySinceLastFlush = true const cfg = this.config.persistence if (this.isReadOnly || cfg?.policy === 'manual') return this._persistDirtyWrites++ @@ -12246,6 +12264,27 @@ export class Brainy implements BrainyInterface { return } + // A CLEAN BRAIN FLUSHES NOTHING, AND SAYS NOTHING. No write has been + // committed since the last flush, so every step below would re-persist + // state identical to what is already on disk — provider flushes, the + // watermark stamps, the generation counter, the entity-tree stamp — and + // print two lines announcing it. On a process holding 21 brains that + // no-op cost 1.26 cores at idle. The witness is set by every committed + // write (see noteWriteForPersistence) and cleared here; a write landing + // DURING this flush sets it again, so it is never lost — the next flush + // does that write's work. + if (!this._dirtySinceLastFlush) { + return + } + this._dirtySinceLastFlush = false + // An explicit flush IS a flush: tell the cadence so, or the very next + // write sees "30s since the last flush" (the cadence only counted its + // own) and kicks a background flush that has nothing left to do, and the + // idle timer fires two seconds later over writes this flush already + // persisted. + this._persistLastFlushAt = Date.now() + this._persistDirtyWrites = 0 + console.log('Flushing Brainy indexes and caches to disk...') const startTime = Date.now() diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index d002164e..ebd3b90c 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1052,6 +1052,17 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { */ private startAutoFlush(): void { this.flushTimer = setInterval(async () => { + // NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an + // index nobody has written to since the last flush has nothing to + // write, and calling into the trees (and their logging) on a cadence + // over a quiet store is exactly the idle cost this law exists to + // remove. + if ( + !this.lsmTreeVerbsBySource.hasPendingWrites() && + !this.lsmTreeVerbsByTarget.hasPendingWrites() + ) { + return + } await this.flush() }, this.config.flushInterval) // Background maintenance must never keep the host process alive — diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts index e19ec145..b4f6052f 100644 --- a/src/graph/lsm/LSMTree.ts +++ b/src/graph/lsm/LSMTree.ts @@ -687,6 +687,17 @@ export class LSMTree { } } + /** + * @description Whether this tree holds anything a flush would write — + * the MemTable is non-empty. Synchronous and O(1), so a background cadence + * can ask before it does anything at all: the engine does no periodic work + * without a cause. + * @returns true when a flush would write; false when it would be a no-op. + */ + hasPendingWrites(): boolean { + return !this.memTable.isEmpty() + } + async close(): Promise { this.stopCompactionTimer() diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts new file mode 100644 index 00000000..8f951d46 --- /dev/null +++ b/tests/integration/idle-costs-nothing.test.ts @@ -0,0 +1,147 @@ +/** + * @module tests/integration/idle-costs-nothing + * @description AN IDLE BRAIN DOES NO WORK. + * + * Measured on a production process holding 21 brains: with no writes for ten + * minutes it printed "All indexes flushed to disk in 216–601ms" per brain + * every ~35 seconds and idled at 1.26 cores. Every one of those flushes + * re-persisted state identical to what was already on disk — the provider + * flushes, the watermark stamps, the generation counter, the entity-tree + * stamp — because `flush()` never asked whether anything had changed. + * + * The laws pinned here: + * (a) the persistence cadence arms only on a write — a brain nobody writes + * to flushes zero times, however long it is left open; + * (b) a flush on a clean brain is O(1): no provider is called, nothing is + * written, and nothing is printed; + * (c) one write earns exactly one flush's worth of work, and no more. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +/** Wait for any in-flight background flush, then let the idle timer settle. */ +async function drainCadence(brain: Brainy): Promise { + const inner = brain as unknown as { _persistBackgroundFlight: Promise | null } + await new Promise((r) => setTimeout(r, 3_000)) + await (inner._persistBackgroundFlight ?? Promise.resolve()) + await new Promise((r) => setTimeout(r, 500)) +} + +/** How long an idle brain is watched. Longer than the 30s flush interval. */ +const IDLE_WATCH_MS = 90_000 + +describe('an idle brain costs nothing', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function openBrain(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-idle-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + return brain + } + + it('flushes zero times over 90 idle seconds, and prints nothing', async () => { + const brain = await openBrain() + // One write and one flush to reach a clean, settled state — then nothing. + await brain.add({ data: 'the only write this test performs', type: NounType.Concept }) + await brain.flush() + + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + + // Watch the providers directly: a flush that runs calls all of them. + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex + const graphIndex = (brain as unknown as { graphIndex: { flush: () => Promise } }).graphIndex + const countsSpy = vi.spyOn(storage, 'flushCounts') + const metadataSpy = vi.spyOn(metadataIndex, 'flush') + const graphSpy = vi.spyOn(graphIndex, 'flush') + + try { + await new Promise((r) => setTimeout(r, IDLE_WATCH_MS)) + } finally { + console.log = origLog + } + + // (a) + (b): nothing ran, nothing was said. + expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) + expect(logged.filter((l) => /Flushing Brainy indexes/.test(l))).toEqual([]) + expect(countsSpy).not.toHaveBeenCalled() + expect(metadataSpy).not.toHaveBeenCalled() + expect(graphSpy).not.toHaveBeenCalled() + }, 180_000) + + it('an explicit flush over a clean brain calls no provider and prints nothing', async () => { + const brain = await openBrain() + await brain.add({ data: 'one write', type: NounType.Concept }) + await brain.flush() // this one does the work + + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex + const countsSpy = vi.spyOn(storage, 'flushCounts') + const metadataSpy = vi.spyOn(metadataIndex, 'flush') + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + try { + await brain.flush() // ...and this one has nothing to do + await brain.flush() + await brain.flush() + } finally { + console.log = origLog + } + + expect(countsSpy).not.toHaveBeenCalled() + expect(metadataSpy).not.toHaveBeenCalled() + expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) + }, 120_000) + + it('one write earns exactly one flush', async () => { + const brain = await openBrain() + await brain.add({ data: 'first', type: NounType.Concept }) + await brain.flush() + // Settle: the first write also kicked a BACKGROUND flush, which is not + // awaited by design. Drain it before counting, or its provider calls land + // inside this test's window and are attributed to the write below. + await drainCadence(brain) + + // Count the flushes that actually RAN. (Provider spies cannot answer this: + // the storage adapter's own count ledger is write-through, so a write calls + // flushCounts() on its own account, with no flush involved.) + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + const ran = () => logged.filter((l) => /All indexes flushed to disk/.test(l)).length + try { + await brain.add({ data: 'second — this is the cause', type: NounType.Concept }) + await brain.flush() + expect(ran()).toBe(1) + + // No further cause, no further work. + await brain.flush() + await brain.flush() + expect(ran()).toBe(1) + } finally { + console.log = origLog + } + }, 120_000) +}) From 131daa08cdc8d5cbb7df8b9f6ec2855946dba52b Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:48:52 -0700 Subject: [PATCH 048/150] feat(open): open never waits for a provider that is rebuilding itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a production store: a metadata provider that had to rebuild made init() pay the ENTIRE rebuild on the foreground — 641 seconds — with every other family idle behind it. The cause is a missing distinction: a provider reporting serving:false because it is BUSY BUILDING ITSELF and one reporting serving:false because it is BROKEN looked identical through healthReport(), and both were answered the same way — call rebuild(), and wait for it. The contract that tells them apart is one optional, synchronous, O(1) hook: `rebuildInProgress(): ProviderRebuildProgress | null`, reporting a phase name and whatever the provider actually measures (done/total/startedAt) — never an estimate dressed as a fact. A provider without the hook behaves exactly as before. With it, a provider owns its own rebuild: - the open gate neither starts a second rebuild nor waits for the provider's, and narrates that it is not waiting and what will refuse meanwhile; - init() returns and every other family serves; - that family's doors refuse BY NAME, carrying the provider's own progress, and say plainly that the door opens by itself and no action is needed — distinct from a broken index, which names repairIndex(); - the epoch stamp does not advance while any family is still being built. Nothing is ever served empty: a not-serving family refuses, as it already did. Pins: tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts — init() returns in milliseconds against a provider claiming a 6s rebuild, brainy starts no rebuild of its own, a filtered read refuses naming the phase and the 4,096/14,056 progress, and the door answers once the provider reports serving. The pin fails loudly rather than vacuously if its stub never installs. --- src/brainy.ts | 85 +++++++++- src/utils/indexReadiness.ts | 80 ++++++++++ ...not-wait-for-a-rebuilding-provider.test.ts | 145 ++++++++++++++++++ 3 files changed, 306 insertions(+), 4 deletions(-) create mode 100644 tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 013885ff..92702364 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -198,7 +198,12 @@ import { import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' -import { assessIndexReadiness, assessProviderHealth } from './utils/indexReadiness.js' +import { + assessIndexReadiness, + assessProviderHealth, + assessProviderRebuild, + describeRebuildProgress +} from './utils/indexReadiness.js' import { reconstructNounWrapper } from './db/factLog.js' import { asBrainyFieldRefusal } from './db/fieldAddressing.js' import { @@ -4540,6 +4545,19 @@ export class Brainy implements BrainyInterface { this._graphAdjacencyVerified = true return 'live' } + // A provider that is REBUILDING ITSELF gets a refusal that says so, + // with its own progress: open deliberately did not wait for it (see + // rebuildIndexesIfNeeded), so this door is temporarily closed and will + // open on its own. Anything else is a broken index needing a repair. + const rebuilding = assessProviderRebuild(this.graphIndex) + if (rebuilding) { + throw new GraphIndexNotReadyError( + `Graph adjacency index is ${describeRebuildProgress(rebuilding)} and is not serving ` + + `yet. find({ connected }), neighbors() and related() refuse rather than serve an ` + + `empty result. The brain is open and every other family is serving; this door opens ` + + `by itself when the provider reports serving — no action is needed.` + ) + } throw new GraphIndexNotReadyError( `Graph adjacency index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. find({ connected }), neighbors() and ` + @@ -4643,6 +4661,15 @@ export class Brainy implements BrainyInterface { this._metadataVerified = true return 'live' } + const rebuilding = assessProviderRebuild(this.metadataIndex) + if (rebuilding) { + throw new MetadataIndexNotReadyError( + `Metadata field index is ${describeRebuildProgress(rebuilding)} and is not serving ` + + `yet. find({ where }) and other filtered reads refuse rather than serve an empty ` + + `result. The brain is open and every other family is serving; this door opens by ` + + `itself when the provider reports serving — no action is needed.` + ) + } throw new MetadataIndexNotReadyError( `Metadata field index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. find({ where }) and other filtered ` + @@ -4772,6 +4799,15 @@ export class Brainy implements BrainyInterface { this._vectorVerified = true return 'live' } + const rebuilding = assessProviderRebuild(this.index) + if (rebuilding) { + throw new VectorIndexNotReadyError( + `Vector index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + + `Semantic find({ query }) and proximity search refuse rather than serve an empty ` + + `result. The brain is open and every other family is serving; this door opens by ` + + `itself when the provider reports serving — no action is needed.` + ) + } throw new VectorIndexNotReadyError( `Vector index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. Semantic find({ query }) and ` + @@ -17144,6 +17180,19 @@ export class Brainy implements BrainyInterface { } if (assessment.readiness === 'not-ready') { + // A provider REBUILDING ITSELF gets a refusal that says so, with its + // own progress: open deliberately did not wait for it, this door is + // temporarily closed, and it opens by itself. Distinct from a broken + // index, which needs an operator. + const rebuilding = assessProviderRebuild(provider) + if (rebuilding) { + throw new ErrorClass( + `${name} index is ${describeRebuildProgress(rebuilding)} and is not serving yet. ` + + `Reads of this family refuse rather than serve an empty result. The brain is open ` + + `and every other family is serving; this door opens by itself when the provider ` + + `reports serving — no action is needed.` + ) + } throw new ErrorClass( `${name} index is not serving (via ${assessment.via}): ` + `${assessment.reasons.join('; ') || 'not ready'}. Reads refuse rather than serve an ` + @@ -17466,9 +17515,37 @@ export class Brainy implements BrainyInterface { // by awaitMigrationLock meanwhile (nothing serves from a half-built index). // Gated per-index, so a non-migrating sibling still rebuilds when it needs // to; a migrating provider is skipped even under epoch-drift or size()===0. - const metadataMigrating = this.providerIsMigrating(this.metadataIndex) - const vectorMigrating = this.providerIsMigrating(this.index) - const graphMigrating = this.providerIsMigrating(this.graphIndex) + // SELF-REBUILD DEFERENCE (the sibling of the migration lock, and the + // reason a production open took 641 seconds): a provider that reports + // `rebuildInProgress()` is ALREADY rebuilding its own index. Brainy must + // neither start a second rebuild nor WAIT for the provider's — init() + // returns, every other family serves, and that family's own doors refuse + // by name (carrying this progress) until the provider reports serving. + // A provider without the hook behaves exactly as before. + const metadataRebuilding = assessProviderRebuild(this.metadataIndex) + const vectorRebuilding = assessProviderRebuild(this.index) + const graphRebuilding = assessProviderRebuild(this.graphIndex) + for (const [leg, progress] of [ + ['metadata', metadataRebuilding], + ['vector', vectorRebuilding], + ['graph', graphRebuilding] + ] as const) { + if (progress) { + prodLog.narrate( + `[Brainy] open(): the ${leg} provider is ${describeRebuildProgress(progress)} — ` + + `open does NOT wait for it. The brain opens now, every other family serves, and ` + + `${leg} reads refuse by name until the provider reports itself serving.` + ) + } + } + + const metadataMigrating = + this.providerIsMigrating(this.metadataIndex) || metadataRebuilding !== null + const vectorMigrating = this.providerIsMigrating(this.index) || vectorRebuilding !== null + const graphMigrating = this.providerIsMigrating(this.graphIndex) || graphRebuilding !== null + // The epoch stamp certifies EVERY derived index, so it must not advance + // while any family is still being built — by a migration lock or by the + // provider itself. const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating // Per-leg decision, in precedence order: a migrating provider owns its diff --git a/src/utils/indexReadiness.ts b/src/utils/indexReadiness.ts index 498f2003..f1b52e3b 100644 --- a/src/utils/indexReadiness.ts +++ b/src/utils/indexReadiness.ts @@ -153,3 +153,83 @@ export function assessProviderHealth(provider: unknown): ProviderHealthAssessmen reasons: readiness === 'not-ready' ? ['isReady() returned false'] : [] } } + +/** + * @description A provider's self-report that it is REBUILDING ITS OWN index + * right now. Returned by the optional `rebuildInProgress()` hook. + * + * The distinction this exists to make: a provider reporting `serving: false` + * because it is BROKEN and a provider reporting `serving: false` because it is + * BUSY BUILDING ITSELF look identical through `healthReport()` alone, and + * brainy treated both the same way — it called `rebuild()` and waited for it, + * on the foreground of `init()`. A production store whose metadata provider + * had to rebuild paid 641 SECONDS of that wait before `init()` returned, with + * every other family idle behind it. + * + * A provider that reports progress here owns its own rebuild: brainy neither + * starts one nor waits for it, `init()` returns, the other families serve, and + * THAT family's doors refuse by name — carrying this progress — until the + * provider reports itself serving. + * + * Every field but `phase` is optional and every field is a MEASUREMENT: a + * provider reports only what it actually tracks, never an estimate dressed as + * a fact. + */ +export interface ProviderRebuildProgress { + /** The provider's own name for what it is doing. Quoted verbatim in refusals. */ + phase: string + /** Units completed so far, if the provider counts them. */ + done?: number + /** Units expected in total, if the provider knows it. */ + total?: number + /** Epoch millis when this rebuild started, if the provider tracks it. */ + startedAt?: number +} + +/** A provider that can report a rebuild it is running itself. */ +interface MaybeRebuildingProvider { + rebuildInProgress?: () => ProviderRebuildProgress | null +} + +/** + * @description Ask a provider whether it is rebuilding itself right now. + * Synchronous, O(1), feature-detected: a provider without the hook reports + * nothing and is treated exactly as before. + * @param provider - Any index provider, or `null`/`undefined`. + * @returns The provider's progress, or `null` when it is not rebuilding (or + * does not implement the hook). + */ +export function assessProviderRebuild(provider: unknown): ProviderRebuildProgress | null { + const p = provider as MaybeRebuildingProvider | null | undefined + if (p == null || typeof p.rebuildInProgress !== 'function') return null + try { + const progress = p.rebuildInProgress() + if (!progress || typeof progress.phase !== 'string' || progress.phase.length === 0) { + return null + } + return progress + } catch { + // A throwing hook says nothing trustworthy about a rebuild; fall through to + // the ordinary health verdict rather than inventing one. + return null + } +} + +/** + * @description Render a rebuild progress report as one operator-facing clause, + * for a refusal message. Includes only what the provider actually measured. + * @param progress - The provider's report. + * @returns A clause such as `rebuilding ("metadata shadow build", 4,096/14,056, 12s elapsed)`. + */ +export function describeRebuildProgress(progress: ProviderRebuildProgress): string { + const parts: string[] = [`"${progress.phase}"`] + if (typeof progress.done === 'number' && typeof progress.total === 'number') { + parts.push(`${progress.done.toLocaleString()}/${progress.total.toLocaleString()}`) + } else if (typeof progress.done === 'number') { + parts.push(`${progress.done.toLocaleString()} done`) + } + if (typeof progress.startedAt === 'number') { + parts.push(`${Math.round((Date.now() - progress.startedAt) / 1000)}s elapsed`) + } + return `rebuilding (${parts.join(', ')})` +} diff --git a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts new file mode 100644 index 00000000..459d7dfc --- /dev/null +++ b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts @@ -0,0 +1,145 @@ +/** + * @module tests/integration/open-does-not-wait-for-a-rebuilding-provider + * @description OPEN DOES NOT WAIT FOR A PROVIDER THAT IS REBUILDING ITSELF. + * + * Measured on a production store: a metadata provider that had to rebuild made + * `init()` pay the ENTIRE rebuild on the foreground — 641 seconds — with every + * other family idle behind it, because a provider reporting `serving: false` + * because it is BUSY BUILDING and one reporting `serving: false` because it is + * BROKEN were indistinguishable, and both were answered the same way: call + * `rebuild()`, and wait. + * + * The law: a provider that reports `rebuildInProgress()` owns its own rebuild. + * `init()` returns; every other family serves; THAT family's doors refuse by + * name, carrying the provider's own progress; and the doors open by themselves + * when the provider reports serving. Nothing is ever served empty. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import type { ProviderRebuildProgress } from '../../src/utils/indexReadiness.js' + +/** How long the stub provider claims to be rebuilding. */ +const REBUILD_MS = 6_000 + +describe('a provider rebuilding itself never blocks open', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + it('init() returns in milliseconds, the family refuses by name, then answers', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-provider-')) + dirs.push(dir) + + // Seed a store so the open has something to (not) rebuild. + const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await seed.init() + await seed.add({ data: 'a row with a plain field', type: NounType.Concept, metadata: { kind: 'report' } }) + await seed.flush() + await seed.close() + + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + + // Dress the metadata index as a provider that is rebuilding ITSELF: not + // serving, and honest about why. `init()` wires the real index first, so + // the hooks are installed on the instance as soon as it exists — the gate + // reads them by feature detection, exactly as it would a native provider's. + const rebuildStartedAt = Date.now() + const stillRebuilding = () => Date.now() - rebuildStartedAt < REBUILD_MS + let rebuildCalls = 0 + + const inner = brain as unknown as { + metadataIndex: Record + setupIndex?: unknown + } + // Install on the prototype-free instance right after construction by + // patching the property the moment init() assigns it. + const install = (target: Record) => { + const realRebuild = target.rebuild as () => Promise + target.rebuildInProgress = (): ProviderRebuildProgress | null => + stillRebuilding() + ? { phase: 'metadata shadow build', done: 4_096, total: 14_056, startedAt: rebuildStartedAt } + : null + target.healthReport = () => ({ + provider: 'metadata', + healthy: !stillRebuilding(), + serving: !stillRebuilding(), + generation: 1, + invariants: [], + unledgered: [] + }) + target.rebuild = async () => { + rebuildCalls++ + return realRebuild.call(target) + } + } + + // init() constructs the metadata index; patch as soon as it exists, before + // the gate consults it. A microtask hop after the index is assigned is + // enough because the gate runs later in the same init. + const initPromise = (async () => { + const originalEnsure = (brain as unknown as { setupIndex?: () => unknown }).setupIndex + void originalEnsure + return brain.init() + })() + // Patch on the first tick the index exists. + const patcher = setInterval(() => { + if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { + install(inner.metadataIndex) + } + }, 1) + const startedAt = Date.now() + try { + await initPromise + } finally { + clearInterval(patcher) + } + const openMs = Date.now() - startedAt + + // If the patch did not land before the gate ran, this test proves nothing — + // say so loudly rather than passing vacuously. + expect( + typeof inner.metadataIndex.rebuildInProgress, + 'the stub provider was never installed — the test is vacuous' + ).toBe('function') + + // 1. The open did not wait out the rebuild. + expect(openMs).toBeLessThan(REBUILD_MS) + // 2. And brainy did not start a rebuild of its own on top of the provider's. + expect(rebuildCalls).toBe(0) + + // 3. The family's door refuses BY NAME, carrying the provider's progress. + let refusal: Error | null = null + try { + await brain.find({ where: { kind: 'report' } } as never) + } catch (err) { + refusal = err as Error + } + expect(refusal, 'a not-serving metadata family must refuse, never serve empty').not.toBeNull() + expect(refusal!.message).toMatch(/metadata shadow build/i) + expect(refusal!.message).toMatch(/4,096\/14,056/) + expect(refusal!.message).toMatch(/no action is needed/i) + + // 4. Other families keep serving — the brain is open. + const all = await brain.getNouns?.({ pagination: { limit: 1 } } as never) + expect(all ?? true).toBeTruthy() + + // 5. When the provider reports itself serving, the door opens by itself. + await new Promise((r) => setTimeout(r, REBUILD_MS)) + ;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false + await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined() + }, 180_000) +}) From 50676c02f44bd3efacaf79fce94adf216c623ff6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:50:26 -0700 Subject: [PATCH 049/150] fix(open): a provider rebuilding itself is a third state, not a CRITICAL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follower to the self-rebuild deference. The open gate's consistency check — "metadata index has 0 entries but storage has N entities" → CRITICAL + a forced second rebuild — knew two states, migrating and not. A provider whose rebuild() returns once the rebuild is OWNED AND RUNNING ONLINE (its doors refusing by name while other families serve) legitimately reports 0 entries there, so every first contact printed a false CRITICAL and kicked a redundant second rebuild. The exemption rides the rebuild-progress hook, NOT isMigrating() — widening that would hold every write and 503 the whole brain through the migration snapshot, which is worse than the false alarm. The check's real class is untouched: a provider reporting 0 entries with no rebuild in progress still trips it. The crash-recovery rebuild kick gets the same deference: a provider already rebuilding itself from canonical is doing exactly that work, and the fold ran in the generation store's open before any provider existed, so what it is reading is the repaired canonical. Pin: a provider stub reporting a rebuild and 0 entries opens with no CRITICAL line and no second rebuild; the vacuous-stub case fails loudly. --- src/brainy.ts | 32 +++++++++++-- ...not-wait-for-a-rebuilding-provider.test.ts | 47 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 92702364..dad609b3 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1489,10 +1489,27 @@ export class Brainy implements BrainyInterface { `[Brainy] Rebuilding indexes after crash recovery rolled back ` + `${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)` ) + // SELF-REBUILD DEFERENCE, same law as the open gate: a provider that + // is already rebuilding itself from canonical is doing exactly this + // work. Kicking a second rebuild on top of it is redundant at best. + // Safe by ordering: the crash-recovery fold ran in the generation + // store's open, BEFORE any provider was constructed, so a provider + // rebuilding now is reading the repaired canonical records. + const kick = async (leg: string, provider: { rebuild: () => Promise }) => { + const rebuilding = assessProviderRebuild(provider) + if (rebuilding) { + prodLog.narrate( + `[Brainy] crash-recovery rebuild: the ${leg} provider is already ` + + `${describeRebuildProgress(rebuilding)} from canonical — not kicking a second one.` + ) + return + } + await provider.rebuild() + } await Promise.all([ - this.metadataIndex.rebuild(), - this.index.rebuild(), - this.graphIndex.rebuild() + kick('metadata', this.metadataIndex), + kick('vector', this.index as unknown as { rebuild: () => Promise }), + kick('graph', this.graphIndex) ]) } @@ -17757,6 +17774,15 @@ export class Brainy implements BrainyInterface { // when the metadata provider holds the migration lock: a 0 count there // reflects its in-place rebuild in progress, not a missed rebuild, so // forcing a second rebuild would collide with the provider's own. + // THREE states, not two. `metadataMigrating` above is true for a + // provider holding the migration lock AND for one that reports it is + // rebuilding itself — a provider whose rebuild() returns once the + // rebuild is OWNED AND RUNNING (online, its doors refusing by name) + // legitimately reports 0 entries here, and calling that CRITICAL would + // print a false alarm and kick a redundant second rebuild on every + // first contact. The check's real class — a rebuild that ran to + // completion and produced nothing — is untouched: a provider reporting + // 0 entries with NO rebuild in progress still trips it. if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) { console.error( `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + diff --git a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts index 459d7dfc..a46ad6a5 100644 --- a/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts +++ b/tests/integration/open-does-not-wait-for-a-rebuilding-provider.test.ts @@ -142,4 +142,51 @@ describe('a provider rebuilding itself never blocks open', () => { ;(brain as unknown as { _metadataVerified: boolean })._metadataVerified = false await expect(brain.find({ where: { kind: 'report' } } as never)).resolves.toBeDefined() }, 180_000) + + it('a rebuilding provider reporting 0 entries is not a CRITICAL, and gets no second rebuild', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-rebuilding-critical-')) + dirs.push(dir) + const seed = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await seed.init() + await seed.add({ data: 'a stored entity', type: NounType.Concept }) + await seed.flush() + await seed.close() + + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + + let rebuildCalls = 0 + const errors: string[] = [] + const origError = console.error + console.error = ((...a: unknown[]) => { errors.push(a.map(String).join(' ')) }) as typeof console.error + + const inner = brain as unknown as { metadataIndex: Record } + const patcher = setInterval(() => { + if (inner.metadataIndex && !inner.metadataIndex.rebuildInProgress) { + const target = inner.metadataIndex + target.rebuildInProgress = () => ({ phase: 'online metadata rebuild', startedAt: Date.now() }) + target.healthReport = () => ({ + provider: 'metadata', healthy: false, serving: false, + generation: 1, invariants: [], unledgered: [] + }) + // The shape the native engine now has: the index reports NOTHING while + // its rebuild runs online behind refusing doors. + target.getStats = async () => ({ totalEntries: 0 }) + target.rebuild = async () => { rebuildCalls++ } + } + }, 1) + try { + await brain.init() + } finally { + clearInterval(patcher) + console.error = origError + } + + expect( + typeof inner.metadataIndex.rebuildInProgress, + 'the stub provider was never installed — the test is vacuous' + ).toBe('function') + expect(errors.filter((l) => /CRITICAL: Metadata index has 0 entries/.test(l))).toEqual([]) + expect(rebuildCalls).toBe(0) + }, 180_000) }) From 48802ba3859b3119c36cf22d85edac559750d6d5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:57:43 -0700 Subject: [PATCH 050/150] feat(contract): declare contract 1, serve three operators, refuse four by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Open Brainy's side of the API contract the accelerated engine published. DECLARED: package.json carries "brainyContract": 1 and the engine states its own via contractVersion() / BRAINY_CONTRACT_VERSION — two engines compare an integer instead of probing prototypes, and a tool reads the package field without importing the engine. Pinned so the two can never drift apart. SERVED: hasAll, noneOf and excludes now work on the index path. The defect underneath was worse than the reported divergence — the metadata index's operator switch had NO DEFAULT CASE, so any operator without a case left the field's match set at its initial [] and find() returned an empty page. Documented, validator-accepted, matcher-implemented operators answering silently wrong. hasAll intersects each element's posting set (an empty operand is vacuously true of every row that has the field), noneOf complements their union, excludes complements contains. REFUSED BY NAME: startsWith, endsWith, matches and length raise INVALID_QUERY naming the operator, the field and the reason. An equality/range posting index cannot evaluate a substring, a pattern or an array length without reading every row — which is the cost this path exists to avoid — so it refuses rather than answering an empty page. Both engines now agree on all 25 tokens and contract 1 has no remaining operator divergence. This is a visible change for a consumer calling those four through find({ where }): an empty page becomes a typed refusal. EMITTED: scripts/emit-contract-manifest.mjs generates docs/api-contract.json from the BUILT surface — prototype doors, exported error classes, the operator sets read out of their single definitions, the field-addressing vocabulary, the health verdicts. Nothing hand-maintained, so a diff between two manifests is a diff between two engines. `--check` fails on a stale manifest, which makes the announce-every-addition duty mechanical rather than remembered. RATIFIED in docs/contract-1-ratification.md: the 41-of-57 required split with the promise spelled out (a refusal is part of a door; deprecation is not removal), the serving-withholding list confirmed exhaustive and identical, the minor/major rule adopted with the announcement duty, the 30 storage seam methods committed as supported surface until Stage 2, and a finding filed against the spec — is / isNot / greaterEqual / lessEqual are listed there as served aliases and have never existed in this engine, which throws INVALID_QUERY on all four. --- docs/api-contract.json | 1545 +++++++++++++++++ package.json | 1 + scripts/emit-contract-manifest.mjs | 129 ++ src/index.ts | 1 + src/neural/embeddedPatterns.ts | 2 +- src/neural/embeddedTypeEmbeddings.ts | 4 +- src/utils/metadataIndex.ts | 89 + src/utils/version.ts | 24 + .../filter-operator-conformance.test.ts | 151 ++ 9 files changed, 1943 insertions(+), 3 deletions(-) create mode 100644 docs/api-contract.json create mode 100644 scripts/emit-contract-manifest.mjs create mode 100644 tests/integration/filter-operator-conformance.test.ts diff --git a/docs/api-contract.json b/docs/api-contract.json new file mode 100644 index 00000000..9dadcc0e --- /dev/null +++ b/docs/api-contract.json @@ -0,0 +1,1545 @@ +{ + "contractVersion": 1, + "engine": "@soulcraftlabs/brainy", + "prose": "docs/contract-1-ratification.md", + "compatibility": { + "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", + "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" + }, + "doors": [ + { + "name": "adaptiveHistoryBudgetBytes", + "kind": "method", + "arity": 1 + }, + { + "name": "add", + "kind": "method", + "arity": 1 + }, + { + "name": "addMany", + "kind": "method", + "arity": 1 + }, + { + "name": "adoptLogAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "adoptLogAuthorityInner", + "kind": "method", + "arity": 0 + }, + { + "name": "aggViewFromEntity", + "kind": "method", + "arity": 1 + }, + { + "name": "anyProviderMigrating", + "kind": "method", + "arity": 0 + }, + { + "name": "applyFusionScoring", + "kind": "method", + "arity": 2 + }, + { + "name": "applyGraphConstraints", + "kind": "method", + "arity": 2 + }, + { + "name": "armIdleFlushTimer", + "kind": "method", + "arity": 2 + }, + { + "name": "asOf", + "kind": "method", + "arity": 2 + }, + { + "name": "assertGenerationStoreReady", + "kind": "method", + "arity": 1 + }, + { + "name": "assertWritable", + "kind": "method", + "arity": 1 + }, + { + "name": "audit", + "kind": "method", + "arity": 0 + }, + { + "name": "auditGraph", + "kind": "method", + "arity": 0 + }, + { + "name": "autoAdoptLegacyVfsBlobsIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "autoAlpha", + "kind": "method", + "arity": 1 + }, + { + "name": "autoCompactHistory", + "kind": "method", + "arity": 0 + }, + { + "name": "awaitMigrationLock", + "kind": "method", + "arity": 1 + }, + { + "name": "awaitPendingEmbeds", + "kind": "method", + "arity": 0 + }, + { + "name": "backfillAggregateIfNeeded", + "kind": "method", + "arity": 1 + }, + { + "name": "batchGet", + "kind": "method", + "arity": 2 + }, + { + "name": "brainWideStrictRequiresSubtype", + "kind": "method", + "arity": 1 + }, + { + "name": "bridgeLegacyPendingEmbedSidecars", + "kind": "method", + "arity": 0 + }, + { + "name": "buildAtGenerationVectors", + "kind": "method", + "arity": 2 + }, + { + "name": "buildGraphView", + "kind": "method", + "arity": 4 + }, + { + "name": "buildMetadataFilter", + "kind": "method", + "arity": 1 + }, + { + "name": "buildMigrationUpdate", + "kind": "method", + "arity": 5 + }, + { + "name": "buildRelationMigrationUpdate", + "kind": "method", + "arity": 5 + }, + { + "name": "cacheVerbInt", + "kind": "method", + "arity": 2 + }, + { + "name": "canServeVectorAtGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "checkHealth", + "kind": "method", + "arity": 0 + }, + { + "name": "checkMigrations", + "kind": "method", + "arity": 0 + }, + { + "name": "clear", + "kind": "method", + "arity": 0 + }, + { + "name": "clearPendingEmbed", + "kind": "method", + "arity": 1 + }, + { + "name": "close", + "kind": "method", + "arity": 0 + }, + { + "name": "closeDurableSteps", + "kind": "method", + "arity": 0 + }, + { + "name": "cluster", + "kind": "method", + "arity": 1 + }, + { + "name": "collectProviderInvariants", + "kind": "method", + "arity": 0 + }, + { + "name": "compactHistory", + "kind": "method", + "arity": 1 + }, + { + "name": "consumeMetadataWatermarkVerdict", + "kind": "method", + "arity": 1 + }, + { + "name": "convertMetadataToEntity", + "kind": "method", + "arity": 2 + }, + { + "name": "convertNounToEntity", + "kind": "method", + "arity": 1 + }, + { + "name": "counts", + "kind": "accessor" + }, + { + "name": "createIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "createMigrationBackupIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "createPinnedDb", + "kind": "method", + "arity": 1 + }, + { + "name": "createResult", + "kind": "method", + "arity": 4 + }, + { + "name": "dbFinalizationRegistry", + "kind": "accessor" + }, + { + "name": "dbHost", + "kind": "accessor" + }, + { + "name": "defineAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "detectIdKind", + "kind": "method", + "arity": 3 + }, + { + "name": "diagnostics", + "kind": "method", + "arity": 0 + }, + { + "name": "diff", + "kind": "method", + "arity": 2 + }, + { + "name": "embed", + "kind": "method", + "arity": 1 + }, + { + "name": "embedBatch", + "kind": "method", + "arity": 2 + }, + { + "name": "emitCommitted", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceSubtypeOnAdd", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceSubtypeOnRelate", + "kind": "method", + "arity": 4 + }, + { + "name": "enforceTrackedFieldValues", + "kind": "method", + "arity": 2 + }, + { + "name": "enhanceNLPResult", + "kind": "method", + "arity": 2 + }, + { + "name": "enqueuePendingEmbed", + "kind": "method", + "arity": 1 + }, + { + "name": "ensureAggregationIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "ensureIndexesLoaded", + "kind": "method", + "arity": 0 + }, + { + "name": "ensureInitialized", + "kind": "method", + "arity": 1 + }, + { + "name": "entityForAggFromRawRecord", + "kind": "method", + "arity": 1 + }, + { + "name": "entityFromGenerationRecord", + "kind": "method", + "arity": 3 + }, + { + "name": "entityIntsToUuids", + "kind": "method", + "arity": 1 + }, + { + "name": "entityViewFromRawRecord", + "kind": "method", + "arity": 2 + }, + { + "name": "excludedVisibilityTiers", + "kind": "method", + "arity": 1 + }, + { + "name": "executeGraphSearch", + "kind": "method", + "arity": 2 + }, + { + "name": "executeProximitySearch", + "kind": "method", + "arity": 1 + }, + { + "name": "executeTextSearch", + "kind": "method", + "arity": 2 + }, + { + "name": "executeVectorSearch", + "kind": "method", + "arity": 3 + }, + { + "name": "explain", + "kind": "method", + "arity": 1 + }, + { + "name": "export", + "kind": "method", + "arity": 0 + }, + { + "name": "extract", + "kind": "method", + "arity": 2 + }, + { + "name": "extractConcepts", + "kind": "method", + "arity": 2 + }, + { + "name": "extractEntities", + "kind": "method", + "arity": 2 + }, + { + "name": "factSegmentPaths", + "kind": "method", + "arity": 1 + }, + { + "name": "fieldCountsAggregateName", + "kind": "method", + "arity": 1 + }, + { + "name": "fillSubtypes", + "kind": "method", + "arity": 1 + }, + { + "name": "filterIdsBelted", + "kind": "method", + "arity": 2 + }, + { + "name": "find", + "kind": "method", + "arity": 1 + }, + { + "name": "findAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "findDuplicates", + "kind": "method", + "arity": 1 + }, + { + "name": "findMatchingWords", + "kind": "method", + "arity": 3 + }, + { + "name": "flush", + "kind": "method", + "arity": 0 + }, + { + "name": "formatInfo", + "kind": "method", + "arity": 0 + }, + { + "name": "formatSubtypeError", + "kind": "method", + "arity": 1 + }, + { + "name": "generation", + "kind": "method", + "arity": 0 + }, + { + "name": "generationDigest", + "kind": "method", + "arity": 1 + }, + { + "name": "get", + "kind": "method", + "arity": 2 + }, + { + "name": "getActivePlugins", + "kind": "method", + "arity": 0 + }, + { + "name": "getAvailableFields", + "kind": "method", + "arity": 0 + }, + { + "name": "getBackgroundDeduplicator", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldsForType", + "kind": "method", + "arity": 1 + }, + { + "name": "getFieldStatistics", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldsWithCardinality", + "kind": "method", + "arity": 0 + }, + { + "name": "getFieldValues", + "kind": "method", + "arity": 1 + }, + { + "name": "getIndexStats", + "kind": "method", + "arity": 0 + }, + { + "name": "getIndexStatus", + "kind": "method", + "arity": 0 + }, + { + "name": "getMemoryStats", + "kind": "method", + "arity": 0 + }, + { + "name": "getNeighborUuids", + "kind": "method", + "arity": 2 + }, + { + "name": "getNounCount", + "kind": "method", + "arity": 0 + }, + { + "name": "getOptimalQueryPlan", + "kind": "method", + "arity": 1 + }, + { + "name": "getStats", + "kind": "method", + "arity": 1 + }, + { + "name": "getStorageType", + "kind": "method", + "arity": 0 + }, + { + "name": "getSubtypeRule", + "kind": "method", + "arity": 1 + }, + { + "name": "getTripleIntelligence", + "kind": "method", + "arity": 0 + }, + { + "name": "getTypedNeighbors", + "kind": "method", + "arity": 4 + }, + { + "name": "getVerbCount", + "kind": "method", + "arity": 0 + }, + { + "name": "graph", + "kind": "accessor" + }, + { + "name": "graphAccelerationProvider", + "kind": "method", + "arity": 0 + }, + { + "name": "graphCommunities", + "kind": "method", + "arity": 1 + }, + { + "name": "graphCommunitiesFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphCommunitiesNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphEntityInt", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExport", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExportFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphExportNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphPath", + "kind": "method", + "arity": 3 + }, + { + "name": "graphPathFallback", + "kind": "method", + "arity": 3 + }, + { + "name": "graphPathNative", + "kind": "method", + "arity": 4 + }, + { + "name": "graphRank", + "kind": "method", + "arity": 1 + }, + { + "name": "graphRankFallback", + "kind": "method", + "arity": 1 + }, + { + "name": "graphRankNative", + "kind": "method", + "arity": 2 + }, + { + "name": "graphSubgraph", + "kind": "method", + "arity": 2 + }, + { + "name": "graphSubgraphFallback", + "kind": "method", + "arity": 4 + }, + { + "name": "graphSubgraphFromQuery", + "kind": "method", + "arity": 5 + }, + { + "name": "graphSubgraphNative", + "kind": "method", + "arity": 5 + }, + { + "name": "groupByLabel", + "kind": "method", + "arity": 2 + }, + { + "name": "hasStorageMethod", + "kind": "method", + "arity": 1 + }, + { + "name": "hasVectorOrTextCriteria", + "kind": "method", + "arity": 1 + }, + { + "name": "health", + "kind": "method", + "arity": 0 + }, + { + "name": "highlight", + "kind": "method", + "arity": 1 + }, + { + "name": "highlightSemanticPhase", + "kind": "method", + "arity": 5 + }, + { + "name": "history", + "kind": "method", + "arity": 2 + }, + { + "name": "historyStats", + "kind": "method", + "arity": 0 + }, + { + "name": "hub", + "kind": "accessor" + }, + { + "name": "hydrateIdMapperForGraphRebuild", + "kind": "method", + "arity": 0 + }, + { + "name": "hydrateNativeSubgraph", + "kind": "method", + "arity": 2 + }, + { + "name": "import", + "kind": "method", + "arity": 2 + }, + { + "name": "importPluginPackage", + "kind": "method", + "arity": 1 + }, + { + "name": "incidentEdges", + "kind": "method", + "arity": 3 + }, + { + "name": "indexStats", + "kind": "method", + "arity": 0 + }, + { + "name": "init", + "kind": "method", + "arity": 1 + }, + { + "name": "insights", + "kind": "method", + "arity": 0 + }, + { + "name": "isEmbeddingReady", + "kind": "method", + "arity": 0 + }, + { + "name": "isInfrastructureWrite", + "kind": "method", + "arity": 1 + }, + { + "name": "isInitialized", + "kind": "accessor" + }, + { + "name": "isReadOnly", + "kind": "accessor" + }, + { + "name": "kickBackgroundFlush", + "kind": "method", + "arity": 1 + }, + { + "name": "kickEmbedWorker", + "kind": "method", + "arity": 0 + }, + { + "name": "legacyLayoutMigrationPhase", + "kind": "method", + "arity": 0 + }, + { + "name": "loadAnalyticsGraph", + "kind": "method", + "arity": 1 + }, + { + "name": "loadPlugins", + "kind": "method", + "arity": 0 + }, + { + "name": "logAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "maintenanceDebt", + "kind": "method", + "arity": 0 + }, + { + "name": "materializeAtGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "metadataIndexRetractionOp", + "kind": "method", + "arity": 3 + }, + { + "name": "migrate", + "kind": "method", + "arity": 1 + }, + { + "name": "migrateField", + "kind": "method", + "arity": 1 + }, + { + "name": "migrateInternal", + "kind": "method", + "arity": 2 + }, + { + "name": "migrateLegacyZeroNormVfsRootIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "migrationSnapshot", + "kind": "method", + "arity": 0 + }, + { + "name": "neededFamiliesMigrating", + "kind": "method", + "arity": 1 + }, + { + "name": "neighbors", + "kind": "method", + "arity": 2 + }, + { + "name": "newId", + "kind": "method", + "arity": 0 + }, + { + "name": "nlp", + "kind": "method", + "arity": 0 + }, + { + "name": "normalizeConfig", + "kind": "method", + "arity": 1 + }, + { + "name": "noteWriteForPersistence", + "kind": "method", + "arity": 0 + }, + { + "name": "now", + "kind": "method", + "arity": 0 + }, + { + "name": "onChange", + "kind": "method", + "arity": 1 + }, + { + "name": "pagination", + "kind": "accessor" + }, + { + "name": "parseMigrationPath", + "kind": "method", + "arity": 1 + }, + { + "name": "parseNaturalQuery", + "kind": "method", + "arity": 1 + }, + { + "name": "pathExists", + "kind": "method", + "arity": 2 + }, + { + "name": "pendingEmbedCount", + "kind": "method", + "arity": 0 + }, + { + "name": "performInit", + "kind": "method", + "arity": 1 + }, + { + "name": "persistPinnedGeneration", + "kind": "method", + "arity": 2 + }, + { + "name": "persistSingleOp", + "kind": "method", + "arity": 6 + }, + { + "name": "pickMetadataProbe", + "kind": "method", + "arity": 1 + }, + { + "name": "pickVectorProbe", + "kind": "method", + "arity": 0 + }, + { + "name": "pinGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "planGetEntity", + "kind": "method", + "arity": 3 + }, + { + "name": "planTransact", + "kind": "method", + "arity": 1 + }, + { + "name": "planTxAdd", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxRelate", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxRemove", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxUnrelate", + "kind": "method", + "arity": 3 + }, + { + "name": "planTxUpdate", + "kind": "method", + "arity": 3 + }, + { + "name": "projectionGauges", + "kind": "method", + "arity": 0 + }, + { + "name": "providerForFamily", + "kind": "method", + "arity": 1 + }, + { + "name": "providerIsMigrating", + "kind": "method", + "arity": 1 + }, + { + "name": "providerMigrationStatus", + "kind": "method", + "arity": 0 + }, + { + "name": "queryAggregate", + "kind": "method", + "arity": 2 + }, + { + "name": "queryIndexFamilies", + "kind": "method", + "arity": 1 + }, + { + "name": "readPath", + "kind": "method", + "arity": 2 + }, + { + "name": "ready", + "kind": "accessor" + }, + { + "name": "rebuildIndexesIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "rebuildMetadataIndexOnline", + "kind": "method", + "arity": 0 + }, + { + "name": "reconcileLogDivergence", + "kind": "method", + "arity": 2 + }, + { + "name": "reconstructPath", + "kind": "method", + "arity": 4 + }, + { + "name": "recordStateAt", + "kind": "method", + "arity": 3 + }, + { + "name": "recoverPendingEmbedsFromLog", + "kind": "method", + "arity": 0 + }, + { + "name": "registerShutdownHooks", + "kind": "method", + "arity": 0 + }, + { + "name": "relate", + "kind": "method", + "arity": 1 + }, + { + "name": "related", + "kind": "method", + "arity": 1 + }, + { + "name": "relateMany", + "kind": "method", + "arity": 1 + }, + { + "name": "relationFromGenerationRecord", + "kind": "method", + "arity": 2 + }, + { + "name": "relationshipSubtypesOf", + "kind": "method", + "arity": 1 + }, + { + "name": "releaseGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "remove", + "kind": "method", + "arity": 1 + }, + { + "name": "removeAggregate", + "kind": "method", + "arity": 1 + }, + { + "name": "removeMany", + "kind": "method", + "arity": 1 + }, + { + "name": "removeMigrationBackupSafe", + "kind": "method", + "arity": 0 + }, + { + "name": "repackHistory", + "kind": "method", + "arity": 1 + }, + { + "name": "repairIndex", + "kind": "method", + "arity": 1 + }, + { + "name": "requestFlush", + "kind": "method", + "arity": 1 + }, + { + "name": "requireProviders", + "kind": "method", + "arity": 1 + }, + { + "name": "requireSubtype", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveAsOfGeneration", + "kind": "method", + "arity": 2 + }, + { + "name": "resolveDiffEndpoint", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveHiddenIds", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveHNSWPersistMode", + "kind": "method", + "arity": 0 + }, + { + "name": "resolveRawGeneration", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveRetentionPolicy", + "kind": "method", + "arity": 0 + }, + { + "name": "resolveVerbEndpointInts", + "kind": "method", + "arity": 1 + }, + { + "name": "resolveVerbIntsToIds", + "kind": "method", + "arity": 1 + }, + { + "name": "restore", + "kind": "method", + "arity": 2 + }, + { + "name": "rrfFusion", + "kind": "method", + "arity": 4 + }, + { + "name": "runAggregationBackfillWalk", + "kind": "method", + "arity": 0 + }, + { + "name": "runAggregationCatchUp", + "kind": "method", + "arity": 0 + }, + { + "name": "runEmbedWorker", + "kind": "method", + "arity": 0 + }, + { + "name": "runOracle", + "kind": "method", + "arity": 1 + }, + { + "name": "runRepairIndexPhases", + "kind": "method", + "arity": 5 + }, + { + "name": "scanFacts", + "kind": "method", + "arity": 1 + }, + { + "name": "seedIdsToInts", + "kind": "method", + "arity": 1 + }, + { + "name": "selectorToSeedIds", + "kind": "method", + "arity": 1 + }, + { + "name": "setRetentionBudget", + "kind": "method", + "arity": 1 + }, + { + "name": "setupEmbedder", + "kind": "method", + "arity": 0 + }, + { + "name": "setupIndex", + "kind": "method", + "arity": 0 + }, + { + "name": "setupStorage", + "kind": "method", + "arity": 0 + }, + { + "name": "similar", + "kind": "method", + "arity": 1 + }, + { + "name": "similarity", + "kind": "method", + "arity": 2 + }, + { + "name": "splitForHighlighting", + "kind": "method", + "arity": 2 + }, + { + "name": "stampBrainFormat", + "kind": "method", + "arity": 0 + }, + { + "name": "stampBrainFormatIfNeeded", + "kind": "method", + "arity": 0 + }, + { + "name": "stampEntityTree", + "kind": "method", + "arity": 0 + }, + { + "name": "stampProjectionWatermarks", + "kind": "method", + "arity": 0 + }, + { + "name": "stats", + "kind": "method", + "arity": 0 + }, + { + "name": "storageAdapter", + "kind": "accessor" + }, + { + "name": "stream", + "kind": "method", + "arity": 0 + }, + { + "name": "streaming", + "kind": "accessor" + }, + { + "name": "subtypesOf", + "kind": "method", + "arity": 1 + }, + { + "name": "trackField", + "kind": "method", + "arity": 1 + }, + { + "name": "transact", + "kind": "method", + "arity": 2 + }, + { + "name": "transactionLog", + "kind": "method", + "arity": 1 + }, + { + "name": "unrelate", + "kind": "method", + "arity": 1 + }, + { + "name": "unvectorNounForRootMigration", + "kind": "method", + "arity": 1 + }, + { + "name": "update", + "kind": "method", + "arity": 1 + }, + { + "name": "updateMany", + "kind": "method", + "arity": 1 + }, + { + "name": "updateRelation", + "kind": "method", + "arity": 1 + }, + { + "name": "upsertMergeParams", + "kind": "method", + "arity": 2 + }, + { + "name": "use", + "kind": "method", + "arity": 1 + }, + { + "name": "usesDefaultWasmEmbedder", + "kind": "method", + "arity": 0 + }, + { + "name": "validateIndexConsistency", + "kind": "method", + "arity": 0 + }, + { + "name": "vectorSearchAtGeneration", + "kind": "method", + "arity": 4 + }, + { + "name": "verbsToRelations", + "kind": "method", + "arity": 1 + }, + { + "name": "verbToRelationLike", + "kind": "method", + "arity": 1 + }, + { + "name": "verifyEntityTreeStamp", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyGraphAdjacencyLive", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyLogAuthority", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyMetadataLive", + "kind": "method", + "arity": 0 + }, + { + "name": "verifyVectorLive", + "kind": "method", + "arity": 0 + }, + { + "name": "versionedIndexProviders", + "kind": "method", + "arity": 0 + }, + { + "name": "vfs", + "kind": "accessor" + }, + { + "name": "waitForIndexed", + "kind": "method", + "arity": 2 + }, + { + "name": "warm", + "kind": "method", + "arity": 0 + }, + { + "name": "warmupEmbeddings", + "kind": "method", + "arity": 0 + }, + { + "name": "warnIfReadsDegraded", + "kind": "method", + "arity": 1 + }, + { + "name": "wireConnectionsCodec", + "kind": "method", + "arity": 0 + }, + { + "name": "wireGraphIdResolver", + "kind": "method", + "arity": 0 + } + ], + "errors": [ + "BrainyError", + "DerivedArtifactMissingError", + "GraphIndexNotReadyError", + "MetadataIndexNotReadyError", + "MigrationInProgressError", + "ProtectedArtifactError", + "VectorIndexNotReadyError" + ], + "operators": { + "accepted": [ + "between", + "contains", + "endsWith", + "eq", + "equals", + "excludes", + "exists", + "greaterThan", + "greaterThanOrEqual", + "gt", + "gte", + "hasAll", + "in", + "length", + "lessThan", + "lessThanOrEqual", + "lt", + "lte", + "matches", + "missing", + "ne", + "noneOf", + "notEquals", + "oneOf", + "startsWith" + ], + "servedOnIndexPath": [ + "between", + "contains", + "eq", + "equals", + "excludes", + "exists", + "greaterThan", + "greaterThanOrEqual", + "gt", + "gte", + "hasAll", + "in", + "lessThan", + "lessThanOrEqual", + "lt", + "lte", + "missing", + "ne", + "noneOf", + "notEquals", + "oneOf" + ], + "refusedByIndexPath": [ + "endsWith", + "length", + "matches", + "startsWith" + ], + "combinators": [ + "allOf", + "anyOf", + "not" + ] + }, + "fieldAddressing": { + "systemKeyPrefix": "system.", + "systemEntityScalars": [ + "confidence", + "createdAt", + "createdBy", + "id", + "service", + "subtype", + "type", + "updatedAt", + "visibility", + "weight" + ], + "systemRelationScalars": [ + "confidence", + "createdAt", + "createdBy", + "service", + "sourceId", + "subtype", + "targetId", + "updatedAt", + "verb", + "visibility", + "weight" + ], + "plumbingFields": [ + "_rev", + "connections", + "data", + "level", + "vector" + ] + }, + "health": { + "verdicts": [ + "pass", + "warn", + "fail" + ], + "healKinds": [ + "none", + "repair", + "rebuild" + ], + "servingWithholdingInvariants": [ + "index-initialized", + "durable-state-present", + "manifest-residency", + "replay-clean", + "strand-latch" + ] + } +} diff --git a/package.json b/package.json index bb6b5a47..60e901de 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "@soulcraftlabs/brainy", "version": "10.4.3", + "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs new file mode 100644 index 00000000..13a9bc6d --- /dev/null +++ b/scripts/emit-contract-manifest.mjs @@ -0,0 +1,129 @@ +#!/usr/bin/env node +/** + * Emit this build's API-contract manifest to docs/api-contract.json. + * + * WHY IT IS GENERATED, NOT WRITTEN: a hand-kept list of doors drifts from the + * code the first time somebody adds one. This reads the surface the build + * actually exposes — the prototype's own methods and accessors, the exported + * error classes, the `where` operator sets, the field-addressing vocabulary, + * the health verdicts — so a diff between two engines' manifests is a diff + * between two engines, never between two authors. + * + * Requirement marking (required / optional per door) is NOT derivable from the + * surface; it is a commitment, and it lives in docs/contract-1-ratification.md. + * This manifest carries the surface; that document carries the promise. + * + * Usage: node scripts/emit-contract-manifest.mjs [--check] + * --check exits non-zero when the committed manifest is stale. + */ + +import { writeFileSync, readFileSync, existsSync } from 'node:fs' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..') +const OUT = join(ROOT, 'docs', 'api-contract.json') + +const { Brainy } = await import(join(ROOT, 'dist', 'brainy.js')) +const errorsModule = await import(join(ROOT, 'dist', 'errors', 'brainyError.js')) +const versionModule = await import(join(ROOT, 'dist', 'utils', 'version.js')) +const fieldAddressing = await import(join(ROOT, 'dist', 'db', 'fieldAddressing.js')) + +/** Every own method and accessor on the class's prototype, minus the private ones. */ +function surfaceOf(ctor) { + const doors = [] + for (const name of Object.getOwnPropertyNames(ctor.prototype)) { + if (name === 'constructor' || name.startsWith('_')) continue + const descriptor = Object.getOwnPropertyDescriptor(ctor.prototype, name) + if (!descriptor) continue + if (typeof descriptor.value === 'function') { + doors.push({ name, kind: 'method', arity: descriptor.value.length }) + } else if (descriptor.get) { + doors.push({ name, kind: 'accessor' }) + } + } + return doors.sort((a, b) => a.name.localeCompare(b.name)) +} + +const errors = Object.entries(errorsModule) + .filter(([name, value]) => typeof value === 'function' && /Error$/.test(name)) + .map(([name]) => name) + .sort() + +// The operator sets, read from the engine's own refusal message so the +// manifest can never disagree with the validator. +const filterSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataFilter.ts'), 'utf-8') +const acceptedMatch = filterSource.match(/const VALUE_OPERATORS = new Set\(\[([\s\S]*?)\]\)/) +if (!acceptedMatch) throw new Error('VALUE_OPERATORS not found — the manifest refuses to guess') +const accepted = [...acceptedMatch[1].matchAll(/'([^']+)'/g)].map((m) => m[1]).sort() + +const indexSource = readFileSync(join(ROOT, 'src', 'utils', 'metadataIndex.ts'), 'utf-8') +const refusedByIndex = ['endsWith', 'length', 'matches', 'startsWith'].filter((op) => + // Proven by the refusal path: these are the tokens with no case in the + // index's operator switch, so they fall to its default and are refused. + !new RegExp(`case '${op}':`).test(indexSource) +) +const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) + +const manifest = { + contractVersion: versionModule.contractVersion(), + engine: '@soulcraftlabs/brainy', + prose: 'docs/contract-1-ratification.md', + compatibility: { + minor: + 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', + major: + 'breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused' + }, + doors: surfaceOf(Brainy), + errors, + operators: { + accepted, + servedOnIndexPath: servedOnIndex, + refusedByIndexPath: refusedByIndex, + combinators: ['allOf', 'anyOf', 'not'] + }, + fieldAddressing: { + systemKeyPrefix: 'system.', + systemEntityScalars: [...(fieldAddressing.SYSTEM_ENTITY_SCALARS ?? [])].sort(), + systemRelationScalars: [...(fieldAddressing.SYSTEM_RELATION_SCALARS ?? [])].sort(), + plumbingFields: [...(fieldAddressing.PLUMBING_FIELDS ?? [])].sort() + }, + health: { + verdicts: ['pass', 'warn', 'fail'], + healKinds: ['none', 'repair', 'rebuild'], + servingWithholdingInvariants: [ + 'index-initialized', + 'durable-state-present', + 'manifest-residency', + 'replay-clean', + 'strand-latch' + ] + } +} + +const rendered = `${JSON.stringify(manifest, null, 2)}\n` + +if (process.argv.includes('--check')) { + if (!existsSync(OUT)) { + console.error(`docs/api-contract.json is missing — run: node scripts/emit-contract-manifest.mjs`) + process.exit(1) + } + if (readFileSync(OUT, 'utf-8') !== rendered) { + console.error( + `docs/api-contract.json is STALE — the public surface changed. Re-emit it and announce ` + + `the addition (minor = additive; a removal is a contract major).` + ) + process.exit(1) + } + console.log(`docs/api-contract.json is current (${manifest.doors.length} doors, contract ${manifest.contractVersion}).`) + process.exit(0) +} + +writeFileSync(OUT, rendered) +console.log( + `Wrote docs/api-contract.json — contract ${manifest.contractVersion}, ` + + `${manifest.doors.length} doors, ${manifest.errors.length} error classes, ` + + `${manifest.operators.accepted.length} operators ` + + `(${manifest.operators.refusedByIndexPath.length} refused by the index path).` +) diff --git a/src/index.ts b/src/index.ts index 973136a5..edc21809 100644 --- a/src/index.ts +++ b/src/index.ts @@ -184,6 +184,7 @@ export { // Export version utilities export { getBrainyVersion } from './utils/version.js' +export { contractVersion, BRAINY_CONTRACT_VERSION } from './utils/version.js' // Export plugin system export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 4f4339f4..92e3057a 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2025-09-29T10:10:00-07:00 + * Generated: 2026-08-27T09:18:45-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index 5b10116c..f4cdd632 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-06-29T10:04:19-07:00 + * Generated: 2026-08-27T09:18:45-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-06-29T10:04:19-07:00", + generatedAt: "2026-08-27T09:18:45-07:00", sizeBytes: { embeddings: 259584, base64: 346112 diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 3cc56b2e..3e0e3d17 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2241,6 +2241,74 @@ export class MetadataIndexManager implements MetadataIndexProvider { break } + // ===== ARRAY SET OPERATORS ===== + // An element-indexed array field makes all three exact on the + // index path. They were previously ABSENT from this switch, so + // `fieldResults` kept its initial `[]` and the whole find() + // returned an empty page — a documented, matcher-implemented + // operator answering silently wrong. Served here instead. + + // hasAll: [a, b] — the field's array contains EVERY operand: + // the intersection of each element's posting set. + case 'hasAll': { + if (!Array.isArray(operand)) { + fieldResults = [] + break + } + if (operand.length === 0) { + // Vacuously true of every row that HAS the field. + const anyBitmap = (this.columnStore && this.columnStore.hasField(field)) + ? await this.columnStore.rangeQuery(field) + : await this.getExistsBitmapLegacy(field) + fieldResults = this.idMapper.intsIterableToUuids(anyBitmap) + break + } + let intersection: Set | null = null + for (const item of operand) { + const ids = new Set(await this.getIds(field, item)) + if (intersection === null) { + intersection = ids + } else { + for (const id of [...intersection]) { + if (!ids.has(id)) intersection.delete(id) + } + } + if (intersection.size === 0) break + } + fieldResults = intersection ? [...intersection] : [] + break + } + + // noneOf: [a, b] — the field's value is NONE of the operands: + // the complement of their union. + case 'noneOf': { + if (!Array.isArray(operand)) { + fieldResults = [] + break + } + const excludeInts: number[] = [] + for (const value of operand) { + for (const uuid of await this.getIds(field, value)) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined) excludeInts.push(intId) + } + } + fieldResults = this.complementIds(excludeInts) + break + } + + // excludes: value — the field's array does NOT contain the value: + // the complement of `contains`. + case 'excludes': { + const excludeInts: number[] = [] + for (const uuid of await this.getIds(field, operand)) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined) excludeInts.push(intId) + } + fieldResults = this.complementIds(excludeInts) + break + } + // ===== MISSING OPERATOR ===== // missing: boolean - equivalent to exists: !boolean case 'missing': { @@ -2257,6 +2325,27 @@ export class MetadataIndexManager implements MetadataIndexProvider { } break } + + // ===== EVERYTHING ELSE: REFUSED BY NAME, NEVER ANSWERED EMPTY ==== + // An equality/range posting index cannot evaluate a substring, a + // pattern or an array length without reading every row, and this + // path exists precisely to avoid that. It used to fall out of the + // switch with `fieldResults` still `[]`, so `find({ where: { name: + // { startsWith: 'a' } } })` returned an empty page and looked like + // an answer. An accepted operator either works or refuses — the + // matcher's own support for these operators governs in-memory + // filtering, never an index-backed find(). + default: + throw new BrainyError( + `Filter operator "${op}" on field "${rawField}" cannot be served by the ` + + `metadata index: an equality/range posting index cannot evaluate substrings, ` + + `patterns or array lengths without reading every row. It is REFUSED rather ` + + `than answered with an empty page. Filter on an indexable operator ` + + `(equals/eq, notEquals/ne, oneOf/in, noneOf, greaterThan/gt, ` + + `greaterThanOrEqual/gte, lessThan/lt, lessThanOrEqual/lte, between, contains, ` + + `excludes, hasAll, exists, missing) and narrow the rest in your own code.`, + 'INVALID_QUERY' + ) } // Intersect this operator's matches with the running set (AND semantics // for multiple operators on the same field). diff --git a/src/utils/version.ts b/src/utils/version.ts index f302eae0..327f923c 100644 --- a/src/utils/version.ts +++ b/src/utils/version.ts @@ -83,3 +83,27 @@ export function getAugmentationVersion(service: string): { augmentation: string; version: getBrainyVersion() } } + +/** + * The API-contract version this build implements — a single integer that two + * engines can compare without probing prototypes. + * + * A MINOR release is ADDITIVE: doors and error codes may be added, never + * removed or narrowed, and the contract integer does not move. A MAJOR release + * is what a REQUIRED door's removal or a behavioural narrowing costs, and it + * bumps this integer. A consumer pinning `brainyContract` in a peer range is + * therefore pinning "what I may call", not "which build I run". + * + * Declared in package.json as `"brainyContract"` so a manifest, a tool, or a + * sibling package can read it without importing the engine, and returned here + * so a running process can state its own. + */ +export const BRAINY_CONTRACT_VERSION = 1 as const + +/** + * @description The API-contract version this build implements. + * @returns The contract integer — see {@link BRAINY_CONTRACT_VERSION}. + */ +export function contractVersion(): number { + return BRAINY_CONTRACT_VERSION +} diff --git a/tests/integration/filter-operator-conformance.test.ts b/tests/integration/filter-operator-conformance.test.ts new file mode 100644 index 00000000..628017e7 --- /dev/null +++ b/tests/integration/filter-operator-conformance.test.ts @@ -0,0 +1,151 @@ +/** + * @module tests/integration/filter-operator-conformance + * @description THE OPERATOR SET, AND WHAT EACH TOKEN DOES ON THE INDEX PATH. + * + * The contract-1 manifest splits this engine's `where` operators three ways — + * served, served-beyond-baseline, refused-by-name — and two engines must agree + * token for token. This lane is the machine-checkable side of that agreement: + * it asserts the EXACT accepted set (so a manifest can be diffed against a run + * rather than against prose), and it pins each of the three classes. + * + * The defect it closes: the metadata index's operator switch had no default + * case, so an operator it does not implement — `hasAll`, `noneOf`, `excludes`, + * `startsWith`, `endsWith`, `matches`, `length` — left the field's match set at + * its initial `[]` and `find()` returned an empty page. A documented operator, + * implemented in the in-memory matcher, answering silently wrong. Three of the + * seven are now SERVED on the index path; the other four are REFUSED BY NAME, + * because an equality/range posting index cannot evaluate a substring, a + * pattern or an array length without reading every row. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { contractVersion, BRAINY_CONTRACT_VERSION } from '../../src/utils/version.js' + +/** The accepted `where` value-operator tokens, as a sorted list. */ +const ACCEPTED_OPERATORS = [ + 'between', 'contains', 'endsWith', 'eq', 'equals', 'excludes', 'exists', + 'greaterThan', 'greaterThanOrEqual', 'gt', 'gte', 'hasAll', 'in', 'length', + 'lessThan', 'lessThanOrEqual', 'lt', 'lte', 'matches', 'missing', 'ne', + 'noneOf', 'notEquals', 'oneOf', 'startsWith' +] as const + +/** Served on the index path with exact posting-set semantics. */ +const SERVED_ON_INDEX = [ + 'between', 'contains', 'eq', 'equals', 'exists', 'greaterThan', + 'greaterThanOrEqual', 'gt', 'gte', 'in', 'lessThan', 'lessThanOrEqual', + 'lt', 'lte', 'missing', 'ne', 'notEquals', 'oneOf', + 'excludes', 'hasAll', 'noneOf' +] as const + +/** Accepted by name, refused by the index path — never answered empty. */ +const REFUSED_BY_INDEX = ['endsWith', 'length', 'matches', 'startsWith'] as const + +describe('filter operator conformance', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + async function seeded(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-operators-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + await brain.add({ + data: 'a document about ferrets', + type: NounType.Document, + metadata: { tags: ['ferret', 'small', 'furry'], team: 'alpha' } + }) + await brain.add({ + data: 'a document about whales', + type: NounType.Document, + metadata: { tags: ['whale', 'large'], team: 'beta' } + }) + await brain.flush() + return brain + } + + it('the accepted operator set is exactly these 25 tokens', async () => { + const brain = await seeded() + // The engine names its own valid set in the refusal it raises for an + // unknown token — the honest place to read it from. + let message = '' + try { + await brain.find({ where: { team: { notIn: ['alpha'] } } } as never) + } catch (err) { + message = (err as Error).message + } + expect(message).toMatch(/Unknown filter operator "notIn"/) + const listed = (message.match(/Valid operators: ([^.]+)\./)?.[1] ?? '') + .split(',') + .map((t) => t.trim()) + .filter(Boolean) + .sort() + expect(listed).toEqual([...ACCEPTED_OPERATORS].sort()) + expect(listed.length).toBe(25) + // Four tokens a sibling manifest listed as served aliases are NOT in this + // engine's set and never have been — they raise INVALID_QUERY. + for (const absent of ['is', 'isNot', 'greaterEqual', 'lessEqual']) { + expect(listed).not.toContain(absent) + await expect( + brain.find({ where: { team: { [absent]: 'alpha' } } } as never) + ).rejects.toThrow(/Unknown filter operator/) + } + }, 120_000) + + it('serves hasAll, noneOf and excludes on the index path — never an empty page', async () => { + const brain = await seeded() + + const hasAll = await brain.find({ where: { tags: { hasAll: ['ferret', 'furry'] } } } as never) + expect(hasAll.length).toBe(1) + expect((hasAll[0] as { metadata?: Record }).metadata?.team).toBe('alpha') + + const noneOf = await brain.find({ where: { team: { noneOf: ['alpha'] } } } as never) + expect(noneOf.length).toBe(1) + expect((noneOf[0] as { metadata?: Record }).metadata?.team).toBe('beta') + + const excludes = await brain.find({ where: { tags: { excludes: 'whale' } } } as never) + expect(excludes.length).toBe(1) + expect((excludes[0] as { metadata?: Record }).metadata?.team).toBe('alpha') + + // hasAll with an operand nothing carries is EMPTY because it is empty — + // the honest zero, reached by evaluating the operator. + const none = await brain.find({ where: { tags: { hasAll: ['ferret', 'whale'] } } } as never) + expect(none.length).toBe(0) + }, 120_000) + + it('refuses the four index-unserveable operators BY NAME', async () => { + const brain = await seeded() + for (const op of REFUSED_BY_INDEX) { + const operand = op === 'length' ? 3 : 'a' + await expect( + brain.find({ where: { team: { [op]: operand } } } as never), + `${op} must refuse, never answer an empty page` + ).rejects.toThrow(new RegExp(`Filter operator "${op}".*cannot be served by the metadata index`, 's')) + } + }, 120_000) + + it('declares its contract version in code and in package.json', async () => { + expect(contractVersion()).toBe(1) + expect(BRAINY_CONTRACT_VERSION).toBe(1) + const pkg = JSON.parse(readFileSync(join(process.cwd(), 'package.json'), 'utf-8')) + expect(pkg.brainyContract).toBe(contractVersion()) + }) + + it('the three classes partition the accepted set', () => { + expect([...SERVED_ON_INDEX, ...REFUSED_BY_INDEX].sort()).toEqual([...ACCEPTED_OPERATORS].sort()) + }) +}) From c1f0972395a908d551560ca7596ea18b9fe95ab5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 10:58:00 -0700 Subject: [PATCH 051/150] chore: keep the generated neural stamps at main's values The build regenerates these from the git commit time; a local rebuild moved only the stamp. Restored so the branch carries no incidental churn. --- src/neural/embeddedPatterns.ts | 2 +- src/neural/embeddedTypeEmbeddings.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 92e3057a..4f4339f4 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-08-27T09:18:45-07:00 + * Generated: 2025-09-29T10:10:00-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index f4cdd632..5b10116c 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-08-27T09:18:45-07:00 + * Generated: 2026-06-29T10:04:19-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-08-27T09:18:45-07:00", + generatedAt: "2026-06-29T10:04:19-07:00", sizeBytes: { embeddings: 259584, base64: 346112 From 4a67aa0fb97da0588083854e870dfd6b0a0e714e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:01:43 -0700 Subject: [PATCH 052/150] perf(vfs): the old-root sweep runs once per store, not once per open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a 14,056-noun / 72,679-verb production-shaped store, measured solo under an exclusive lock: the vfs-bootstrap phase cost 43,021 ms of a cold open and 52,696 ms of a WARM REOPEN. What dominates it is a migration sweep — a filtered find() over the whole store hunting for root directories created before the fixed root id existed. A store either carries such duplicates or never will, and the sweep ran on every open, forever, in the foreground. It is now caused by the store's state instead of by the open count: a durable marker under _system/ records that the sweep has run, and a store carrying it never sweeps again. A store without one sweeps in the BACKGROUND — the sweep only removes duplicate roots, nothing serves from them, and it was already declared non-critical — narrated at both ends, with whenRootSweepSettled() for anyone who needs to observe rather than race it. An adapter with no raw-object door keeps the old behaviour: correctness over cost, never a silent skip. Pins: tests/integration/vfs-root-sweep-once.test.ts — the sweep runs on the first open and never on the second or third; a sweep slowed to 4s does not delay the open. --- src/vfs/VirtualFileSystem.ts | 105 ++++++++++++++++- tests/integration/vfs-root-sweep-once.test.ts | 106 ++++++++++++++++++ 2 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 tests/integration/vfs-root-sweep-once.test.ts diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 59a16be4..19470b48 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -6,6 +6,7 @@ */ import { Readable, Writable } from 'stream' +import { prodLog } from '../utils/logger.js' import crypto from 'crypto' import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' @@ -66,6 +67,15 @@ export class VirtualFileSystem implements IVirtualFileSystem { private config: Required> & { rootEntityId?: string } private rootEntityId?: string private initialized = false + /** + * The one-time old-root sweep, in flight. See {@link sweepOldRootsIfNeeded}. + */ + private rootSweep?: Promise + /** + * Where the completed old-root sweep is recorded. Engine plumbing under + * `_system/`, like every other marker there — never enumerated as data. + */ + private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json' private currentUser: string = 'system' // Track current user for collaboration // Knowledge Layer features available via augmentation (brain.use('knowledge')) @@ -143,8 +153,17 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Create or find root entity this.rootEntityId = await this.initializeRoot() - // Clean up old UUID-based roots (one-time migration) - await this.cleanupOldRoots() + // Clean up old UUID-based roots — ONCE PER STORE, BEHIND THE DOORS. + // This is a migration sweep for roots created before the fixed root id + // existed. It ran on EVERY open, forever: a filtered find over the whole + // store hunting for duplicates that a store has either always had or + // never will. MEASURED on a 14,056-noun / 72,679-verb store: the phase it + // dominates cost 43-53 SECONDS of every open, warm reopens included. + // Now: a durable marker records that the sweep has run, and a store + // carrying it never sweeps again; a store without one sweeps in the + // BACKGROUND (the sweep only removes duplicate roots — nothing serves + // from them — and it was always declared non-critical). + this.rootSweep = this.sweepOldRootsIfNeeded() // Initialize projection registry with auto-discovery of built-in projections this.projectionRegistry = new ProjectionRegistry() @@ -394,6 +413,88 @@ export class VirtualFileSystem implements IVirtualFileSystem { * * This is a one-time migration helper that can be removed in future versions. */ + /** + * @description Run the old-root sweep at most once per store, in the + * background, and record that it ran. See the call site in {@link init} for + * the measurement that made this necessary. + * @returns A promise that settles when the sweep has finished (or was + * skipped); nothing in the read path awaits it. + */ + private async sweepOldRootsIfNeeded(): Promise { + const store = this.rawObjectStore() + if (store === null) { + // A storage adapter with no raw-object door cannot carry the marker. + // Sweep every open, as before — correctness over cost. + await this.cleanupOldRoots() + return + } + try { + const marker = await store.readRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH) + if (marker !== null && marker !== undefined) return + } catch { + // Unreadable marker: sweep, and rewrite it below. + } + prodLog.narrate( + '[VFS] one-time sweep for pre-fixed-id root directories running in the background — ' + + 'the open does not wait for it, and once it has run this store never sweeps again.' + ) + const startedAt = Date.now() + await this.cleanupOldRoots() + try { + await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, { + sweptAt: new Date().toISOString(), + durationMs: Date.now() - startedAt + }) + prodLog.narrate( + `[VFS] old-root sweep complete in ${Date.now() - startedAt}ms and recorded — ` + + 'no future open pays for it.' + ) + } catch (error) { + // Unrecorded sweep = the next open sweeps again. Conservative, and said + // out loud rather than quietly repeated forever. + prodLog.narrate( + `[VFS] old-root sweep finished in ${Date.now() - startedAt}ms but could NOT be ` + + `recorded (${(error as Error).message}) — the next open will sweep again.` + ) + } + } + + /** + * @description Settle once the background old-root sweep has finished. + * Resolves immediately when the store already carried the marker. Exists so + * tests and operators can observe the sweep instead of racing it; no read + * path waits on it. + * @returns A promise that settles with the sweep. + */ + public async whenRootSweepSettled(): Promise { + await this.rootSweep + } + + /** + * @description The brain's storage adapter, narrowed to the raw-object door + * this migration marker needs. Boundary: `Brainy.storage` is private, and + * this is the same reach-in the engine uses elsewhere for exactly this kind + * of engine-internal artifact. Returns null when the adapter has no + * raw-object door. + */ + private rawObjectStore(): { + readRawObject: (key: string) => Promise + writeRawObject: (key: string, value: unknown) => Promise + } | null { + const storage = (this.brain as unknown as { storage?: Record }).storage + if ( + storage && + typeof storage.readRawObject === 'function' && + typeof storage.writeRawObject === 'function' + ) { + return storage as unknown as { + readRawObject: (key: string) => Promise + writeRawObject: (key: string, value: unknown) => Promise + } + } + return null + } + private async cleanupOldRoots(): Promise { try { // Find any old VFS roots with UUID-based IDs (not our fixed ID) diff --git a/tests/integration/vfs-root-sweep-once.test.ts b/tests/integration/vfs-root-sweep-once.test.ts new file mode 100644 index 00000000..f84f4413 --- /dev/null +++ b/tests/integration/vfs-root-sweep-once.test.ts @@ -0,0 +1,106 @@ +/** + * @module tests/integration/vfs-root-sweep-once + * @description THE OLD-ROOT SWEEP RUNS ONCE PER STORE, NOT ONCE PER OPEN. + * + * The VFS bootstrap ran a filtered `find()` over the whole store on EVERY + * open, hunting for root directories created before the fixed root id existed + * — duplicates a store has either always had or never will. MEASURED on a + * 14,056-noun / 72,679-verb store: the phase it dominates cost 43–53 SECONDS + * of every open, warm reopens included. + * + * The law: a migration sweep is caused by the store's state, not by the clock + * or the open count. It runs behind the doors, records that it ran, and a + * store carrying that record never sweeps again. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' + +describe('the VFS old-root sweep', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function open(dir: string): Promise { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + return brain + } + + it('sweeps on the first open, records it, and never sweeps again', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-')) + dirs.push(dir) + + const sweepSpy = vi.spyOn( + VirtualFileSystem.prototype as unknown as { cleanupOldRoots: () => Promise }, + 'cleanupOldRoots' + ) + + const first = await open(dir) + await (first.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).toHaveBeenCalledTimes(1) + // The record is durable engine plumbing under _system/, like every other marker. + expect( + existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || + existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) + ).toBe(true) + + await first.add({ data: 'a row so the store is not trivially empty', type: NounType.Concept }) + await first.flush() + await first.close() + brains.splice(brains.indexOf(first), 1) + + sweepSpy.mockClear() + const second = await open(dir) + await (second.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).not.toHaveBeenCalled() + + await second.close() + brains.splice(brains.indexOf(second), 1) + + // ...and a third open, to prove it is the record and not a one-off. + sweepSpy.mockClear() + const third = await open(dir) + await (third.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + expect(sweepSpy).not.toHaveBeenCalled() + }, 180_000) + + it('the open does not wait for the sweep', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-async-')) + dirs.push(dir) + + const proto = VirtualFileSystem.prototype as unknown as Record< + string, + (...args: unknown[]) => Promise + > + const real = proto.cleanupOldRoots + proto.cleanupOldRoots = async function slow(this: unknown, ...args: unknown[]) { + await new Promise((r) => setTimeout(r, 4_000)) + return real.apply(this, args) + } + try { + const startedAt = Date.now() + const brain = await open(dir) + const openMs = Date.now() - startedAt + expect(openMs).toBeLessThan(3_000) + await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + } finally { + proto.cleanupOldRoots = real + } + }, 180_000) +}) From 5a091ccad95628368470a76ce5cebe30274b0651 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:02:57 -0700 Subject: [PATCH 053/150] feat(open): the open names the STEP that cost the time, not just the phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A phase that costs a minute and names only itself tells an operator where to look but not what to look at. MEASURED on a real 14,056-noun / 72,679-verb store, the warm reopen's generation-store phase cost 55,538 ms with nothing inside it named — the fold was skipped (the close was clean), so the cost was somewhere else entirely and the breakdown could not say where. Six steps inside the open now report their own wall with their own cause when they exceed the phase threshold: the generation store's open (manifest, committed ranges, fact log, packed tier, crash replay), the entity-tree stamp verification, the brain-format read, the pre-upgrade backup, the derived-index gate, and the VFS init. Silent under the threshold, so a fast open says nothing extra. Same always-visible channel as the phase lines. --- src/brainy.ts | 56 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index dad609b3..66317322 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1163,6 +1163,23 @@ export class Brainy implements BrainyInterface { ) }, OPEN_HEARTBEAT_MS) if (typeof openHeartbeat.unref === 'function') openHeartbeat.unref() + /** + * Narrate one STEP inside a phase when it turns out to be expensive. + * A phase that costs a minute and names only itself tells an operator + * where to look but not what to look at; this names the step. Silent + * under OPEN_PHASE_NARRATE_MS, so a fast open says nothing extra. + */ + const step = async (name: string, cause: string, run: () => Promise): Promise => { + const startedAt = Date.now() + try { + return await run() + } finally { + const elapsed = Date.now() - startedAt + if (elapsed >= OPEN_PHASE_NARRATE_MS) { + prodLog.narrate(`[Brainy] open: step "${name}" took ${elapsed}ms — ${cause}`) + } + } + } const markPhase = (name: string): void => { const now = Date.now() const elapsed = now - lastPhaseCheckpoint @@ -1263,9 +1280,12 @@ export class Brainy implements BrainyInterface { // instances skip recovery (readers never write; the next writer // repairs). this.generationStore = new GenerationStore(this.storage) - const generationOpenResult = await this.generationStore.open({ - readOnly: this.config.mode === 'reader' - }) + const generationOpenResult = await step( + 'generation-store.open', + 'reading the generation manifest and committed ranges, opening the fact log and the ' + + 'packed segment tier, and folding any crash-recovery replay', + () => this.generationStore.open({ readOnly: this.config.mode === 'reader' }) + ) // The generation fact log is CANONICAL state, not a derived index — no // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare @@ -1303,7 +1323,11 @@ export class Brainy implements BrainyInterface { // rollup invariants against the log head + live counters. Loud on // genuine incoherence (repairIndex heals), silent on absent/coherent, // benign-behind refreshes at the next flush. Never blocks open. - await this.verifyEntityTreeStamp() + await step( + 'verify-entity-tree-stamp', + 'comparing the entity tree\'s stamped generation and rollups against the store', + () => this.verifyEntityTreeStamp() + ) // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — @@ -1315,7 +1339,11 @@ export class Brainy implements BrainyInterface { // them from the canonical records and then re-stamps the marker AFTER the // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old / // absent marker, so the next open idempotently re-rebuilds). - this._brainFormat = await readBrainFormat(this.storage) + this._brainFormat = await step( + 'read-brain-format', + 'reading the on-disk format marker that decides whether the derived indexes are stale', + () => readBrainFormat(this.storage) + ) this._indexEpochStale = this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH @@ -1326,7 +1354,11 @@ export class Brainy implements BrainyInterface { // upgrade verifies + stamps; retained on failure. No-op for a reader, for // non-filesystem storage, or for a brain with no persisted data. if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) { - await this.createMigrationBackupIfNeeded() + await step( + 'pre-upgrade-backup', + 'snapshotting the brain directory before a one-time format rebuild (migrationBackup)', + () => this.createMigrationBackupIfNeeded() + ) } // PHASE 2 of 5 — "generation-store open+fold": GenerationStore @@ -1606,7 +1638,11 @@ export class Brainy implements BrainyInterface { // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true // once this open-time step has run. - await this.rebuildIndexesIfNeeded() + await step( + 'rebuild-indexes-if-needed', + 'the derived-index gate: each family\'s readiness verdict, and any build it asks for', + () => this.rebuildIndexesIfNeeded() + ) this.lazyRebuildCompleted = true // Check for pending data migrations @@ -1679,7 +1715,11 @@ export class Brainy implements BrainyInterface { // Initialize VFS: Ensure VFS is ready when accessed as property // This eliminates need for separate vfs.init() calls - zero additional complexity this._vfs = new VirtualFileSystem(this) - await this._vfs.init() + await step( + 'vfs.init', + 'creating or adopting the VFS root and wiring the path resolver', + () => this._vfs!.init() + ) this._vfsInitialized = true // Mark VFS as fully initialized // 8.0 MVCC: infrastructure bootstrap (VFS root, etc.) is now the From e4c27fbca81569d2f870ed873f2908263bab9b7e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:06:06 -0700 Subject: [PATCH 054/150] fix(flush): clear() and repairIndex() set the dirty witness themselves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both mutate durable state outside the two commit paths, so neither was seen by the flush witness added with the idle-flush law. A clear() followed by a flush() would have found the brain "clean" and skipped the entity-tree stamp, leaving a stamp describing the population the clear had just removed — a false divergence warning at the next open. Closing the gap where it is, rather than widening the witness to guess. --- src/brainy.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/brainy.ts b/src/brainy.ts index 66317322..711d8d02 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8523,6 +8523,11 @@ export class Brainy implements BrainyInterface { */ async clear(): Promise { await this.ensureInitialized() + // A clear mutates durable state without going through a commit path, so + // it must set the dirty witness itself — otherwise a `clear()` followed by + // `flush()` would find the brain "clean" and skip the entity-tree stamp, + // leaving a stamp that describes the population this call just removed. + this._dirtySinceLastFlush = true // Clear storage await this.storage.clear() @@ -18394,6 +18399,10 @@ export class Brainy implements BrainyInterface { */ async repairIndex(options?: { rebuild?: Array<'metadata' | 'graph' | 'vector'> | 'all' }): Promise { await this.ensureInitialized() + // A repair recounts, prunes and rebuilds outside the commit paths; the + // dirty witness is set so a caller's flush after a repair does its normal + // work rather than finding the brain "clean". + this._dirtySinceLastFlush = true const startedAt = Date.now() const families: RepairFamilyReport[] = [] From 9dd399216b1b99c59e5add3f44d850c5d8a5e5b9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:09:05 -0700 Subject: [PATCH 055/150] perf(generations): discover generations by directory name, not by walking the log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a production-shaped store (14,056 nouns / 72,679 verbs, an 11 GB generation history), measured solo under an exclusive lock: the generation-store phase cost 55,538 ms of a WARM REOPEN after a clean close — with the fold correctly skipped, so nothing in that phase's name explained it. This is what it was doing. Discovering which generations exist on disk called listRawObjects('_generations'), which RECURSES the whole tree and returns every file in every generation directory — to extract a set of integers that the top-level directory NAMES already spell out. The cost scales with the entire history, is paid on every open, warm or cold, and grows for the life of the store. A one-level door — listRawPrefixes(prefix), the immediate child directory names — is added to the storage seam. The filesystem adapter answers it with a single readdir; BaseStorage derives it from the recursive listing, so an adapter without a cheap implementation is never wrong, only never faster; and the generation store falls back to the old listing when the door is absent. One behavioural difference, stated: an EMPTY generation directory is now discovered where the file listing could not see it. Above the committed watermark that is a crash scar, and recovery already has an explicit branch for it ("indeterminate partial dir" — dropped, narrated). Below it, it becomes a resolvable generation holding no records, which is what an empty generation means. Suites: the durability kill matrix (15), db-mvcc (30), history repacking (4), rollback trapdoor (3), entity-tree stamp (4) and the full unit suite (2,105) all green. --- src/db/generationStore.ts | 27 ++++++++++++++++++----- src/db/types.ts | 15 +++++++++++++ src/storage/adapters/fileSystemStorage.ts | 24 ++++++++++++++++++++ src/storage/baseStorage.ts | 23 +++++++++++++++++++ 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index d925c9e0..fd052c31 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -537,12 +537,29 @@ export class GenerationStore { this.horizonGen = finiteGen(manifest?.horizon, 'manifest horizon') this.counter = Math.max(finiteGen(counterFile?.generation, 'generation counter'), this.committed) - // Discover existing generation record directories. - const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) + // Discover existing generation record directories — BY DIRECTORY NAME. + // This used to call listRawObjects(), which recurses the whole + // `_generations/` tree and returns every file in every generation, to + // extract a set of integers the top-level directory names already spell. + // MEASURED on a real store with an 11 GB generation history: the phase + // this sits in cost 55,538 ms of a WARM REOPEN after a clean close, with + // no fold to blame — this walk is what it was doing. An adapter without + // the one-level door falls back to the recursive listing, unchanged. const seenGens = new Set() - for (const p of recordPaths) { - const gen = parseGenerationFromPath(p) - if (gen !== null) seenGens.add(gen) + const oneLevel = ( + this.storage as { listRawPrefixes?: (prefix: string) => Promise } + ).listRawPrefixes + if (typeof oneLevel === 'function') { + for (const name of await oneLevel.call(this.storage, GENERATIONS_PREFIX)) { + const gen = Number(name) + if (Number.isSafeInteger(gen) && gen >= 0) seenGens.add(gen) + } + } else { + const recordPaths = await this.storage.listRawObjects(GENERATIONS_PREFIX) + for (const p of recordPaths) { + const gen = parseGenerationFromPath(p) + if (gen !== null) seenGens.add(gen) + } } let rolledBack = 0 diff --git a/src/db/types.ts b/src/db/types.ts index 363de086..56bdef11 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -450,6 +450,21 @@ export interface GenerationStorage { deleteRawObject(path: string): Promise /** List raw object paths under a prefix (normalized, `.gz`-stripped). */ listRawObjects(prefix: string): Promise + /** + * OPTIONAL: the IMMEDIATE child directory names under a prefix — one level, + * no recursion, no file paths. + * + * Why it exists: discovering which generations are on disk needs only the + * top-level directory NAMES under `_generations/`, but the only door for it + * was `listRawObjects`, which recurses the whole tree and returns every file + * in every generation. On a store with a long history that is a full walk of + * the entire generation log, paid on EVERY open, to learn a set of integers + * the directory names already spell out. + * + * An adapter without this door keeps working — the caller falls back to the + * recursive listing. + */ + listRawPrefixes?(prefix: string): Promise /** Remove every object under a prefix (and the directory itself on disk). */ removeRawPrefix(prefix: string): Promise /** Durability barrier: fsync the given object paths (no-op in memory). */ diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 784ea5d8..3f1055c2 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -686,6 +686,30 @@ export class FileSystemStorage extends BaseStorage { return pruned } + /** + * @description The IMMEDIATE child directory names under a prefix — ONE + * `readdir`, no recursion, no file paths. See the seam's JSDoc + * (`src/db/types.ts`) for what this replaced: discovering the generations on + * disk walked the entire generation log on every open, reading out every + * file in every generation, to learn the set of integers the top-level + * directory names already spell. + * @param prefix - Storage-root-relative directory prefix. + * @returns The child directory names (not paths); empty when the prefix does + * not exist. + */ + public override async listRawPrefixes(prefix: string): Promise { + await this.ensureInitialized() + const fullPath = path.join(this.rootDir, prefix) + try { + const entries = await fs.promises.readdir(fullPath, { withFileTypes: true }) + return entries.filter((e: { isDirectory: () => boolean }) => e.isDirectory()) + .map((e: { name: string }) => e.name) + } catch (error: any) { + if (error?.code === 'ENOENT') return [] + throw error + } + } + /** * Primitive operation: List objects under path prefix * All metadata operations use this internally via base class routing diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index f518e68f..d8bcb780 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1437,6 +1437,29 @@ export abstract class BaseStorage extends BaseStorageAdapter { return this.listObjectsUnderPath(prefix) } + /** + * @description The IMMEDIATE child directory names under a prefix — one + * level, no recursion. See the seam's JSDoc (`db/types.ts`) for why a + * separate door exists. This default derives them from the recursive + * listing, so it is never WRONG, only never faster; the filesystem adapter + * overrides it with a single directory read. + * @param prefix - Storage-root-relative directory prefix. + * @returns The child directory names (not paths), in listing order. + */ + public async listRawPrefixes(prefix: string): Promise { + await this.ensureInitialized() + const paths = await this.listObjectsUnderPath(prefix) + const normalizedPrefix = prefix.endsWith('/') ? prefix : `${prefix}/` + const names = new Set() + for (const p of paths) { + const rest = p.startsWith(normalizedPrefix) ? p.slice(normalizedPrefix.length) : null + if (rest === null) continue + const slash = rest.search(/[/\\]/) + if (slash > 0) names.add(rest.slice(0, slash)) + } + return [...names] + } + /** * Remove every object under a storage-root-relative prefix. The filesystem * adapter overrides this with a recursive directory removal; this default From 417ddb5143c7aa4bd33068cdc1a0050a4394ebc7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:13:24 -0700 Subject: [PATCH 056/150] perf(open): answer "are there any entities?" with one directory read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 7.x-to-8.0 layout probe runs on the open path of every store that does not yet carry its completion marker — a restore, a store built by an older release — and asked whether the canonical tree holds anything by LISTING it: a recursive walk of every file in every entity directory, to learn a boolean. It now asks the one-level door added for generation discovery, falling back to the listing on an adapter that lacks it. Also files a defect found while ratifying the operator set: the VFS builds its path-prefix filter as `$startsWith`, an operator no engine spelling accepts, so vfs.searchFiles({ path }) throws INVALID_QUERY on every call that passes a path. Pre-existing, unrelated to the operator work, and left as a filing — a path-prefix search needs a design answer, not a spelling correction. --- src/brainy.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 711d8d02..c479fcc3 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -16732,8 +16732,19 @@ export class Brainy implements BrainyInterface { if (legacyEntityPaths.length === 0) { // Already flat (root entities, no head-branch entities) → stamp the marker // so future opens short-circuit. A genuinely empty/fresh dir gets no marker. - const rootEntities = await probe.listRawObjects('entities') - if (rootEntities.length > 0) { + // "Are there any entities?" is answered by ONE directory read, not by a + // recursive listing of every file in the tree: this runs on the open path + // of every store that does not yet carry the marker (a restore, a store + // built by an older release), and on a large store that listing walks the + // whole canonical tree to learn a boolean. + const oneLevel = ( + probe as unknown as { listRawPrefixes?: (prefix: string) => Promise } + ).listRawPrefixes + const hasRootEntities = + typeof oneLevel === 'function' + ? (await oneLevel.call(probe, 'entities')).length > 0 + : (await probe.listRawObjects('entities')).length > 0 + if (hasRootEntities) { await probe.writeRawObject('_system/migration-layout.json', { layout: 'flat-v8', version: 8, From fb1da1c56dbe8acca7d8cfb6fb66e4446205eaac Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:25:47 -0700 Subject: [PATCH 057/150] perf(idle): the flush-request watch is event-driven; the heartbeat is observability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three idle-burn items from the steady-state audit, and one correction. THE FLUSH-REQUEST WATCH (the strongest of them). It readdir'd the request directory every 500 ms, per brain, for the life of every writer — armed on every non-reader brain whether or not any inspector process existed. In a process holding many stores that is tens of directory reads per second on a completely idle service, plus a stale-request GC on every one of them. It now uses fs.watch, so the arrival itself wakes it and a request is seen SOONER than the poll saw it. Two concessions ride along, both stated in the code: a 30s safety sweep (fs.watch drops events on some network and fuse filesystems, and the GC needs a tick of its own — two orders of magnitude fewer reads than the poll made), and a fall back to the original 500 ms poll, narrated, on a filesystem that cannot watch at all, because an inspector whose request is never seen waits forever. THE WRITER HEARTBEAT goes 10s → 60s. It is observability ONLY — staleness is decided by pid liveness and the fence compares pid + hostname, so no decision anywhere reads the timestamp — and at 10s it was a lock-file write every ten seconds per brain forever, for a value nothing computes with. An operator still sees a heartbeat inside the minute. THE HEALTH NARRATION dedupes by CONTENT, not by the provider's generation counter. That counter bumps on every ledger mutation and rebuild boundary, so a provider bumping it on routine work re-emitted the same unchanged line on every read, while one that never bumped could suppress a line whose reasons had genuinely changed. The generation is still reported; it no longer decides whether the line is worth saying. CORRECTION, and it is against my own earlier claim: the idle-flush commit read a reported idle-CPU observation (many stores, no writes, a flush every ~35s, over a core burned) as caused by the flush path. That does not follow — this engine's cadence is write-driven (every trigger runs through noteWriteForPersistence, which only a committed write calls), so something was CALLING flush() on those brains and the caller is still unidentified. The clean-flush gate makes such a call free; it does not account for it. The code comments and the idle lane now say exactly that. Pins: tests/integration/flush-watcher-event-driven.test.ts — an idle writer makes at most one request-directory read in 8 seconds (the old poll made ~16), and a dropped request is still acked well inside the safety sweep. --- src/brainy.ts | 54 +++++--- src/storage/adapters/fileSystemStorage.ts | 116 +++++++++++++++--- .../flush-watcher-event-driven.test.ts | 94 ++++++++++++++ tests/integration/idle-costs-nothing.test.ts | 17 ++- 4 files changed, 245 insertions(+), 36 deletions(-) create mode 100644 tests/integration/flush-watcher-event-driven.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index c479fcc3..3426dcce 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -749,12 +749,18 @@ export class Brainy implements BrainyInterface { * Whether a write has been committed since the last flush that ran. THE * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written * to has nothing to make durable, and a flush over it must cost nothing and - * say nothing. Measured on a production process holding 21 brains: with no - * writes for ten minutes it still printed "All indexes flushed to disk in - * 216–601ms" per brain every ~35s and idled at 1.26 cores, because a flush - * called every provider, stamped the watermarks, persisted the generation - * counter and re-stamped the entity tree whether or not anything had - * changed. + * say nothing. Before this, a flush called every provider, stamped the + * watermarks, persisted the generation counter and re-stamped the entity + * tree whether or not anything had changed — roughly 28 writes for a store + * that had not moved. + * + * WHAT THIS DOES NOT EXPLAIN, stated so nobody reads it as solved: a + * production process holding 21 brains printed "All indexes flushed to disk + * in 216-601ms" per brain every ~35s and idled at 1.26 cores with no writes + * for ten minutes. This engine's cadence is WRITE-DRIVEN — every trigger + * runs through noteWriteForPersistence, which only a committed write calls — + * so something was calling flush() on those brains, and this gate makes such + * a call free rather than accounting for it. The caller is still unidentified. */ private _dirtySinceLastFlush = false private _persistIdleTimer: ReturnType | null = null @@ -851,7 +857,18 @@ export class Brainy implements BrainyInterface { // Read-gate narration dedup: a degraded-but-serving or not-ready health // report narrates via prodLog.warn ONCE per (provider, report.generation) — // never once per read. Keyed on the provider instance itself. - private _lastNarratedHealthGeneration = new Map() + /** + * The last health narration emitted per provider, keyed by its CONTENT. + * + * This used to dedupe on the provider's `generation` counter, which bumps on + * every ledger mutation and every rebuild boundary — so a provider that + * bumps its generation on routine work re-emitted the same unchanged health + * line on every read that consulted it, and a provider that never bumped + * could suppress a line whose reasons had genuinely changed. The dedupe key + * is now what the line SAYS: an unchanged verdict is silent however the + * generation moves, and a changed verdict is always heard. + */ + private _lastNarratedHealth = new Map() constructor(config?: BrainyConfig) { // The reserved-field write policy died with the field-addressing law: @@ -12366,11 +12383,11 @@ export class Brainy implements BrainyInterface { // committed since the last flush, so every step below would re-persist // state identical to what is already on disk — provider flushes, the // watermark stamps, the generation counter, the entity-tree stamp — and - // print two lines announcing it. On a process holding 21 brains that - // no-op cost 1.26 cores at idle. The witness is set by every committed + // print two lines announcing it. The witness is set by every committed // write (see noteWriteForPersistence) and cleared here; a write landing // DURING this flush sets it again, so it is never lost — the next flush - // does that write's work. + // does that write's work. This makes an unexplained flush FREE; it does + // not explain one (see _dirtySinceLastFlush). if (!this._dirtySinceLastFlush) { return } @@ -17243,12 +17260,17 @@ export class Brainy implements BrainyInterface { if (assessment.reasons.length > 0 && assessment.report != null) { const generation = assessment.report.generation - if (this._lastNarratedHealthGeneration.get(provider) !== generation) { - this._lastNarratedHealthGeneration.set(provider, generation) - prodLog.warn( - `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + - assessment.reasons.join('; ') - ) + // Dedupe by CONTENT, not by the provider's generation counter — see + // _lastNarratedHealth. The generation is still REPORTED (an operator + // wants to know which generation produced the verdict); it just no + // longer decides whether the line is worth saying. + const line = + `[Brainy] ${assessment.report.provider} health (generation ${generation}): ` + + assessment.reasons.join('; ') + const key = `${assessment.report.provider}\u0000${assessment.reasons.join('; ')}` + if (this._lastNarratedHealth.get(provider) !== key) { + this._lastNarratedHealth.set(provider, key) + prodLog.warn(line) } } diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 3f1055c2..9d04b46d 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -107,7 +107,23 @@ export class FileSystemStorage extends BaseStorage { * "the previous writer died" without inferring either from a pid. */ private static readonly WRITER_CLOSE_FILE = '_writer.close' - private static readonly WRITER_HEARTBEAT_MS = 10_000 + /** + * How often the lock file's `lastHeartbeat` is rewritten. + * + * THIS IS OBSERVABILITY ONLY, and the cadence follows from that. Staleness + * is decided by PID LIVENESS alone (see isWriterLockStale) and the fence + * compares pid + hostname — no decision anywhere reads this timestamp. It + * exists so an operator inspecting a lock file, or reading the + * BRAINY_WRITER_LOCKED error, can judge liveness themselves. + * + * At 10s it was a lock-file WRITE every ten seconds per brain, forever: 2.1 + * writes/s across a production process holding 21 idle brains, for a + * human-readable timestamp nothing computes with. At 60s an operator still + * sees a heartbeat inside the minute, at a sixth of the cost. With the + * clean-close record now recording orderly releases explicitly, the + * heartbeat carries even less weight than it did. + */ + private static readonly WRITER_HEARTBEAT_MS = 60_000 private static readonly WRITER_STALE_THRESHOLD_MS = 60_000 private writerLockHeartbeat?: NodeJS.Timeout private writerLockInfo?: WriterLockInfo @@ -135,9 +151,16 @@ export class FileSystemStorage extends BaseStorage { private static readonly FLUSH_REQUEST_DIR = '_flush_requests' private static readonly FLUSH_RESPONSE_DIR = '_flush_responses' private static readonly FLUSH_WATCH_INTERVAL_MS = 500 + /** + * The safety sweep behind the fs.watch: catches events an exotic filesystem + * dropped, and runs the stale-request GC. See startFlushRequestWatcher. + */ + private static readonly FLUSH_SAFETY_SWEEP_MS = 30_000 private static readonly FLUSH_POLL_INTERVAL_MS = 100 private static readonly FLUSH_REQUEST_TTL_MS = 60_000 private flushWatcherInterval?: NodeJS.Timeout + /** The inotify-backed watch on the request directory, when the FS supports one. */ + private flushWatcher?: import('node:fs').FSWatcher private flushWatcherInFlight = false private flushWatcherOnRequest?: () => Promise @@ -2385,36 +2408,101 @@ export class FileSystemStorage extends BaseStorage { /** * Start watching for cross-process flush requests. Called by Brainy.init() - * in writer mode. Polls `locks/_flush_requests/` every - * FLUSH_WATCH_INTERVAL_MS — each new `.req` file triggers the supplied - * callback (`brain.flush()`), after which an `.ack` is written to - * `locks/_flush_responses/` with the same request ID. Stale `.req` files - * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on every tick. + * in writer mode. Each new `.req` file in `locks/_flush_requests/` triggers + * the supplied callback (`brain.flush()`), after which an `.ack` is written + * to `locks/_flush_responses/` with the same request ID. Stale `.req` files + * (>FLUSH_REQUEST_TTL_MS) are garbage-collected on each sweep. + * + * THE WATCH IS EVENT-DRIVEN, NOT A POLL. It used to `readdir` the request + * directory every 500 ms, per brain, for the entire life of every writer — + * armed on every non-reader brain whether or not any inspector process + * existed. MEASURED on a production process holding 21 brains: 42 directory + * reads per second on a completely idle service, plus a stale-request GC + * pass on every one of them. The engine does no periodic work without a + * cause, and a request that has not been made is not a cause. + * + * `fs.watch` (inotify on Linux) delivers the arrival itself, so a request is + * seen SOONER than the old poll saw it. Two honest concessions ride with it: + * - a slow SAFETY SWEEP (FLUSH_SAFETY_SWEEP_MS) still runs, because + * `fs.watch` can miss events on network and fuse filesystems and because + * the stale-request GC needs some tick of its own. At 30s that is 0.7 + * reads/s across 21 brains where the poll cost 42. + * - a filesystem that cannot watch at all falls back to the ORIGINAL + * 500 ms poll, narrated once, because correctness outranks idle cost: + * an inspector whose request is never seen waits forever. */ public override startFlushRequestWatcher(onRequest: () => Promise): void { - if (this.flushWatcherInterval) return // already watching + if (this.flushWatcherInterval || this.flushWatcher) return // already watching this.flushWatcherOnRequest = onRequest const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) const ackDir = path.join(this.lockDir, FileSystemStorage.FLUSH_RESPONSE_DIR) - // Ensure both dirs exist up front so the first .req drop doesn't race with mkdir. - this.ensureDirectoryExists(reqDir).catch(() => {}) - this.ensureDirectoryExists(ackDir).catch(() => {}) - - this.flushWatcherInterval = setInterval(() => { - if (this.flushWatcherInFlight) return // skip overlapping tick + const sweep = (): void => { + if (this.flushWatcherInFlight) return // skip overlapping sweep this.flushWatcherInFlight = true this.processFlushRequests(reqDir, ackDir).finally(() => { this.flushWatcherInFlight = false }) - }, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) + } + + // Ensure both dirs exist up front so the first .req drop doesn't race with + // mkdir — and so there is a directory to watch. + void this.ensureDirectoryExists(reqDir) + .then(() => this.ensureDirectoryExists(ackDir)) + .then(() => { + if (this.flushWatcherOnRequest !== onRequest) return // stopped meanwhile + try { + const watcher = fs.watch(reqDir, () => sweep()) + this.flushWatcher = watcher + watcher.on('error', (err: Error) => { + // A watch that dies mid-life must not leave the door deaf. + console.warn( + `[brainy] Flush-request watch failed (${err.message}) — falling back to polling.` + ) + this.flushWatcher?.close() + this.flushWatcher = undefined + this.startFlushRequestPolling(sweep) + }) + if (typeof watcher.unref === 'function') watcher.unref() + // The safety sweep: missed events on exotic filesystems, and the + // stale-request GC. + this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_SAFETY_SWEEP_MS) + if (typeof this.flushWatcherInterval.unref === 'function') { + this.flushWatcherInterval.unref() + } + // One sweep now: a request may have been dropped before the watch armed. + sweep() + } catch (err) { + console.warn( + `[brainy] Flush-request directory cannot be watched on this filesystem ` + + `(${(err as Error).message}) — polling every ` + + `${FileSystemStorage.FLUSH_WATCH_INTERVAL_MS}ms instead.` + ) + this.startFlushRequestPolling(sweep) + } + }) + .catch(() => { + // The request directory could not be created; nothing to watch. A + // cross-process flush request cannot be made either, so there is + // nothing to miss. + }) + } + + /** The original 500 ms poll — the fallback when a directory cannot be watched. */ + private startFlushRequestPolling(sweep: () => void): void { + if (this.flushWatcherInterval) return + this.flushWatcherInterval = setInterval(sweep, FileSystemStorage.FLUSH_WATCH_INTERVAL_MS) if (typeof this.flushWatcherInterval.unref === 'function') { this.flushWatcherInterval.unref() } } public override stopFlushRequestWatcher(): void { + if (this.flushWatcher) { + this.flushWatcher.close() + this.flushWatcher = undefined + } if (this.flushWatcherInterval) { clearInterval(this.flushWatcherInterval) this.flushWatcherInterval = undefined diff --git a/tests/integration/flush-watcher-event-driven.test.ts b/tests/integration/flush-watcher-event-driven.test.ts new file mode 100644 index 00000000..4b2e80c4 --- /dev/null +++ b/tests/integration/flush-watcher-event-driven.test.ts @@ -0,0 +1,94 @@ +/** + * @module tests/integration/flush-watcher-event-driven + * @description THE FLUSH-REQUEST WATCH IS EVENT-DRIVEN. + * + * It used to `readdir` the request directory every 500 ms, per brain, for the + * life of every writer — armed on every non-reader brain whether or not any + * inspector process existed. MEASURED on a production process holding 21 + * brains: 42 directory reads per second on a completely idle service, plus a + * stale-request GC pass on every one of them. + * + * The law: a request that has not been made is not a cause. The arrival itself + * wakes the watcher, so the request is seen SOONER than the poll saw it, and a + * slow safety sweep covers filesystems that drop watch events and the GC. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs' +import * as nodeFs from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +describe('the flush-request watcher', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function openWriter(): Promise<{ brain: Brainy; dir: string }> { + const dir = mkdtempSync(join(tmpdir(), 'brainy-flush-watch-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + await brain.add({ data: 'a row', type: NounType.Concept }) + await brain.flush() + return { brain, dir } + } + + it('does not poll the request directory on an idle writer', async () => { + const { dir } = await openWriter() + const reqDir = join(dir, 'locks', '_flush_requests') + + // Count real reads of the request directory over a window far longer than + // the old 500ms poll (which would have made ~16 of them). + const realReaddir = nodeFs.promises.readdir + let requestDirReads = 0 + const spy = vi + .spyOn(nodeFs.promises, 'readdir') + .mockImplementation((async (p: unknown, ...rest: unknown[]) => { + if (String(p) === reqDir) requestDirReads++ + return (realReaddir as unknown as (...a: unknown[]) => Promise)(p, ...rest) + }) as typeof nodeFs.promises.readdir) + + await new Promise((r) => setTimeout(r, 8_000)) + spy.mockRestore() + + // The old poll: 500ms → ~16 reads. The safety sweep is 30s → 0 in this window. + expect(requestDirReads).toBeLessThanOrEqual(1) + }, 120_000) + + it('answers a request that arrives, without waiting for the sweep', async () => { + const { brain, dir } = await openWriter() + const reqDir = join(dir, 'locks', '_flush_requests') + const ackDir = join(dir, 'locks', '_flush_responses') + mkdirSync(reqDir, { recursive: true }) + + // Drop a request exactly as an out-of-process inspector does. + const id = 'test-request-0001' + writeFileSync(join(reqDir, `${id}.req`), JSON.stringify({ at: Date.now() })) + + // The ack must land far sooner than the 30s safety sweep. + const deadline = Date.now() + 10_000 + let acked = false + while (Date.now() < deadline) { + try { + const entries = await nodeFs.promises.readdir(ackDir) + if (entries.some((e) => e.startsWith(id))) { acked = true; break } + } catch { /* dir not created yet */ } + await new Promise((r) => setTimeout(r, 100)) + } + expect(acked, 'the watcher must answer an arriving request').toBe(true) + void brain + }, 120_000) +}) diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts index 8f951d46..b5c386cf 100644 --- a/tests/integration/idle-costs-nothing.test.ts +++ b/tests/integration/idle-costs-nothing.test.ts @@ -2,12 +2,17 @@ * @module tests/integration/idle-costs-nothing * @description AN IDLE BRAIN DOES NO WORK. * - * Measured on a production process holding 21 brains: with no writes for ten - * minutes it printed "All indexes flushed to disk in 216–601ms" per brain - * every ~35 seconds and idled at 1.26 cores. Every one of those flushes - * re-persisted state identical to what was already on disk — the provider - * flushes, the watermark stamps, the generation counter, the entity-tree - * stamp — because `flush()` never asked whether anything had changed. + * A flush used to re-persist state identical to what was already on disk — + * the provider flushes, the watermark stamps, the generation counter, the + * entity-tree stamp, roughly 28 writes — because `flush()` never asked whether + * anything had changed. + * + * The field observation that started this: a production process holding 21 + * brains printed "All indexes flushed to disk in 216–601ms" per brain every + * ~35 seconds and idled at 1.26 cores, with no writes for ten minutes. This + * engine's cadence is WRITE-DRIVEN, so that observation is NOT explained by + * the cadence and is not claimed to be fixed here — what is fixed is that such + * a call now costs nothing. Who was calling flush() remains open. * * The laws pinned here: * (a) the persistence cadence arms only on a write — a brain nobody writes From 16d2e1a97ec9ba435a682b91c84b88d4e87bfd95 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:30:00 -0700 Subject: [PATCH 058/150] fix(storage): the flush watcher cannot arm twice in its async window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Arming is asynchronous — the request directory is created before it can be watched — so during that window neither the watcher nor the sweep interval exists yet and the guard let a second call through, leaving two watchers and two sweeps for the life of the store. The callback is the flag that covers the window. --- src/storage/adapters/fileSystemStorage.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 9d04b46d..fa0715df 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -2432,7 +2432,12 @@ export class FileSystemStorage extends BaseStorage { * an inspector whose request is never seen waits forever. */ public override startFlushRequestWatcher(onRequest: () => Promise): void { - if (this.flushWatcherInterval || this.flushWatcher) return // already watching + // Already watching — or already ARMING. The arm is asynchronous (the + // request directory is created before it can be watched), so neither the + // watcher nor the interval exists yet during that window; the callback is + // the flag that covers it. Without this a second call in the window would + // leave two watchers and two sweeps running for the life of the store. + if (this.flushWatcherInterval || this.flushWatcher || this.flushWatcherOnRequest) return this.flushWatcherOnRequest = onRequest const reqDir = path.join(this.lockDir, FileSystemStorage.FLUSH_REQUEST_DIR) From 5c22f9500ce9628e7652613babefb00e094adedd Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:31:57 -0700 Subject: [PATCH 059/150] fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The safety sweep is armed alongside the watch, and startFlushRequestPolling() declines to arm over an existing interval — so when a watch died mid-life the fallback did nothing and the store quietly answered flush requests on a 30s cadence instead of the 500ms one the door promises. The sweep is cleared first. A degrade nobody asked for is still a degrade. --- src/storage/adapters/fileSystemStorage.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index fa0715df..5ec1d88e 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -2467,6 +2467,15 @@ export class FileSystemStorage extends BaseStorage { ) this.flushWatcher?.close() this.flushWatcher = undefined + // The SAFETY sweep must go first. It is already armed at 30s, and + // startFlushRequestPolling() declines to arm over an existing + // interval — so leaving it would quietly leave this store answering + // flush requests on a 30s cadence instead of the 500ms one the door + // promises. A degrade nobody asked for is still a degrade. + if (this.flushWatcherInterval) { + clearInterval(this.flushWatcherInterval) + this.flushWatcherInterval = undefined + } this.startFlushRequestPolling(sweep) }) if (typeof watcher.unref === 'function') watcher.unref() From 2cf3801007a18a0add42b0b6cfe75c245bde556d Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 11:56:26 -0700 Subject: [PATCH 060/150] feat(open): name the two steps that hold the vfs-bootstrap phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED on a 14,056-noun / 72,679-verb production-shaped store, measured solo under an exclusive lock: the vfs-bootstrap phase costs 37.8s on main and 38.0s on this branch — unchanged — and NO "vfs.init" step line was emitted at all, meaning the VFS's own init fell under the 2s narration threshold. The phase is therefore almost entirely NOT the VFS, and the old-root sweep this branch moved to the background was never what made it expensive. What else lives in that span is now named: the log-authority artifact read, the adoption ORACLE (which verifies the log against canonical before flipping a brain to durable-at-ack), the legacy pending-embed sidecar bridge, and the pending-embed recovery fold. One of those holds ~38 seconds of every open of this store and the next measurement will say which, by name, instead of leaving a phase label to be guessed at. --- src/brainy.ts | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 3426dcce..70d46973 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1760,7 +1760,11 @@ export class Brainy implements BrainyInterface { const storedArtifact = await this.storage .readRawObject(LOG_AUTHORITY_PATH) .catch(() => null) - const authority = await readLogAuthority(this.storage) + const authority = await step( + 'read-log-authority', + 'reading the stored storage-authority artifact', + () => readLogAuthority(this.storage) + ) this._logAuthority = authority if (authority.authority === 'log') { this.generationStore.setLogDurability('at-ack') @@ -1771,7 +1775,12 @@ export class Brainy implements BrainyInterface { this.generationStore.getFactLog() !== null ) { try { - await this.adoptLogAuthority() + await step( + 'adopt-log-authority', + 'the adoption oracle: verifying the log against canonical before flipping this ' + + 'brain to durable-at-ack, and backfilling any curable divergence', + () => this.adoptLogAuthority() + ) prodLog.info( '[Brainy] storage authority adopted at open: generation log ' + '(fleet default; oracle green; durable-at-ack enabled)' @@ -1811,8 +1820,16 @@ export class Brainy implements BrainyInterface { // this is where it lands. if (!this.isReadOnly) { try { - await this.bridgeLegacyPendingEmbedSidecars() - await this.recoverPendingEmbedsFromLog() + await step( + 'bridge-pending-embed-sidecars', + 'migrating any pre-log deferred-embed marker files into the generation log', + () => this.bridgeLegacyPendingEmbedSidecars() + ) + await step( + 'recover-pending-embeds', + 'folding the generation log\'s deferred-embed markers back into the pending set', + () => this.recoverPendingEmbedsFromLog() + ) if (this._pendingEmbedIds.size > 0) { prodLog.info( `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + From 02c61636370387f6c3ebc38be1087f58e0a14033 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:04:49 -0700 Subject: [PATCH 061/150] docs: measurements in public history carry numbers, not provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A release audit found hostnames, store identities and operational anecdotes in this branch's commit messages — not trade secrets, but nothing a public repository's permanent history should carry either. The messages were rewritten to keep every number and drop every provenance; the rule is written down here so the next measurement does not have to be caught by an audit. --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 50860cb5..84bd25e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,6 +57,13 @@ see `package.json` for `test:integration`, `test:coverage`, and friends. description states a number, cite the benchmark that produced it (see [docs/performance-envelopes.md](docs/performance-envelopes.md) for the pattern). Don't state an estimate as if it were measured. +- **Measurements carry numbers, not provenance.** Public commit messages and + docs give the SHAPE a number was taken at and never where it was taken: no + hostnames, no store or deployment identities, no operational anecdotes about + someone's running system. "A 14,056-noun / 72,679-verb production-shaped + store, measured solo under an exclusive lock" tells a reader everything the + number depends on; the machine it ran on and whose data it was tell them + nothing except where somebody's infrastructure lives. ## License From 61a469270e476e3ae64755069a39794303bfafd9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:05:46 -0700 Subject: [PATCH 062/150] =?UTF-8?q?docs(releases):=2010.4.4=20consumer=20n?= =?UTF-8?q?otes=20=E2=80=94=20correctness=20and=20observability,=20with=20?= =?UTF-8?q?the=20performance=20line=20stated=20exactly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- RELEASES.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index 4d716efa..e8833b80 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -31,6 +31,115 @@ is sometimes cited as a 7.x removal — those methods never existed on 7.x; the --- +## v10.4.4 — 2026-08-28 + +**A correctness and observability release.** The headline is not speed: it is that a +restart now tells you the truth about itself, a store stops lying about how much it +holds, and the engine stops doing work nobody asked for. There is a performance +improvement and it is modest; it is stated exactly below rather than rounded up. + +### The dark restart — fixed at the root + +A service could stop cleanly, exit 0, having awaited `close()` on every store it held, +and its next boot would announce `Overwriting stale writer lock … appears dead` for +every one of them. Nothing had crashed. Two deployments hit this; the same defect also +made those boots pay a crash-recovery fold they did not owe. + +The cause was not the lock. `close()` released it correctly — when it got there. A +failure part-way through close skipped both the release AND the clean-shutdown marker, +and "the recorded pid is gone" reads identically for an orderly restart and a crash. + +- `close()` is now two parts and the second is unconditional: the flush-request watcher, + the **writer lock**, the VFS timers and the terminal `closed` flag are released whether + the durable steps succeeded or not. The original failure is narrated with what it costs + the next open, then rethrown. +- Releasing the lock writes a **clean-close record** naming the lock generation it gave + up. The next open reads that record instead of guessing: recorded → nothing to recover; + absent → it says so, and names the recovery it is about to run. This also ends two + long-standing false alarms — a recycled pid locking a store out of its own reopen, and + `Re-acquiring writer lock … this is a bug` after a perfectly clean close. +- The signal path stopped failing in a batch. One store's failing flush used to strand + every remaining store's lock and markers — at exit code 0. Now: per-store isolation, the + generation store's close (the marker) is part of shutdown, the lock goes in a `finally`, + and the handler no longer calls `process.exit()` when the host application has its own + signal handler, a race that truncated the host's own shutdown mid-flight. + +### The count ledger stops lying, and `counts.json` is written atomically + +The all-tier scalars are the denominator a coverage check subtracts against. A ledger +derived under the old rule — one entity per id DIRECTORY — counted ghost and scar +containers as rows, and was only FLAGGED suspect: it went on serving wrong numbers for +the life of the store. Two copies of one archive could disagree, and a downstream index +heal reported remaining work that did not exist. + +- Such a ledger now derives itself honestly **in the background** after the open, counting + identity records, and persists the correction stamped. Nothing waits for it, because no + read is served from a denominator. +- A derivation that raced a write refuses to stamp its number: one retry on a quiet store, + then the ledger stays SUSPECT and names `repairIndex()` as the door that recounts under + a barrier. +- `counts.json` is written temp+rename. A truncating write left a window in which a + concurrent reader saw the file EMPTY — and an unparseable ledger sends the next open + down the full-rescan path, so the cheapest file in the store was buying the most + expensive recovery. + +### An open and a repair narrate themselves — on a channel a log level cannot silence + +A store could open for three minutes and print nothing at all. The phase timings existed; +they were written to a channel that every production-looking environment clamps away. + +- Narration moved to an always-visible channel. An open now heartbeats the phase it is in, + names each phase as it ends with what it was paying for, and names the expensive STEP + inside a phase. `repairIndex()` does the same and its receipt carries a per-family + `durationMs` — a repair that ran for half an hour with no output could only be watched + through `top`. +- A brain nobody has written to now does nothing: a flush over a clean store is a no-op + and says nothing, the graph index's auto-flush asks before it acts, and the + cross-process flush-request watch is **event-driven** (`fs.watch`) instead of polling a + directory every 500 ms per store forever, with a slow safety sweep behind it and a + narrated fall back to polling where a filesystem cannot be watched. +- A provider that is REBUILDING ITSELF is no longer confused with a broken one. `init()` + does not wait for it, every other family serves, and that family's doors refuse **by + name, carrying the provider's own progress**, saying plainly that they open by + themselves and no action is needed. Health narration dedupes by content, so an unchanged + verdict is silent however a provider's generation counter moves. + +### For operators — one behaviour change + +**Four `where` operators that previously returned an empty page now raise +`INVALID_QUERY`:** `startsWith`, `endsWith`, `matches` and `length`. An equality/range +posting index cannot evaluate a substring, a pattern or an array length without reading +every row, and it now refuses by name instead of answering with an empty result that +looks like an answer. + +**Three that previously returned an empty page are now SERVED:** `hasAll`, `noneOf` and +`excludes`. All 25 accepted operator tokens now agree between this engine and its +accelerated counterpart. + +### Performance — stated exactly + +Measured on a 14,056-noun / 72,679-verb production-shaped store, both builds solo under +an exclusive lock: + +- **Warm reopen after a clean close: 85.7 s → 77.0 s (−10.2%).** The whole of that gain is + one fix — generation discovery reads directory NAMES instead of recursively walking the + entire generation log (−9.2 s, and it scales with history rather than row count). The + VFS phase is **unchanged**. +- **Cold open: −31.4 s** (518.1 s → 486.7 s), of which the count-ledger derivation moving + off the critical path accounts for storage-init dropping 5,941 ms → 25 ms. +- **A dominant ~38 s remains, diagnosed and NOT fixed.** It is not the VFS — the VFS's own + init is under 2 s of that phase. It is the log-authority adoption and/or the + pending-embed log recovery, both now instrumented so the next measurement names the + culprit outright. + +Continuing work, named so nobody has to rediscover it: that ~38 s term; making the +generation store's committed-range set lazy; the hydration path that substitutes +`Date.now()` for an unreadable stored timestamp (inventing data); and a VFS path-prefix +filter built with a `$startsWith` spelling no operator set accepts, so +`searchFiles({ path })` throws today. + +--- + ## v10.4.3 — 2026-08-27 (Open Brainy's first release) **`@soulcraftlabs/brainy` 10.4.3 is the same engine as `@soulcraft/brainy` 10.4.2, byte for From a8c724a202a9bad3ac6d5da20e08209478af2f2d Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:08:19 -0700 Subject: [PATCH 063/150] docs: the contract manifest stands alone; public docs describe this engine only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest's prose pointer named a document that answers a confidential specification, and such a document does not belong in a public repository even in summary. The pointer is dropped — the manifest is generated from this engine's own surface and is self-describing — and the requirement marking it deliberately omits is recorded with the contract's owner rather than here. The standard is written down so this is not relitigated per document. --- CONTRIBUTING.md | 4 ++++ docs/api-contract.json | 1 - scripts/emit-contract-manifest.mjs | 5 ++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 84bd25e3..54d4f784 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,10 @@ see `package.json` for `test:integration`, `test:coverage`, and friends. store, measured solo under an exclusive lock" tells a reader everything the number depends on; the machine it ran on and whose data it was tell them nothing except where somebody's infrastructure lives. +- **Documents that answer or reference a confidential specification never enter + this repository, even summarized.** The public docs describe THIS engine and + the published contract, and nothing else — a summary of a private document is + still that document's contents. ## License diff --git a/docs/api-contract.json b/docs/api-contract.json index 9dadcc0e..aafd838a 100644 --- a/docs/api-contract.json +++ b/docs/api-contract.json @@ -1,7 +1,6 @@ { "contractVersion": 1, "engine": "@soulcraftlabs/brainy", - "prose": "docs/contract-1-ratification.md", "compatibility": { "minor": "additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms", "major": "breaking — a door removed, an answer narrowed, an ordering law changed, an optional door promoted to required, or an operator moved from served to refused" diff --git a/scripts/emit-contract-manifest.mjs b/scripts/emit-contract-manifest.mjs index 13a9bc6d..be73d4ca 100644 --- a/scripts/emit-contract-manifest.mjs +++ b/scripts/emit-contract-manifest.mjs @@ -10,8 +10,8 @@ * between two engines, never between two authors. * * Requirement marking (required / optional per door) is NOT derivable from the - * surface; it is a commitment, and it lives in docs/contract-1-ratification.md. - * This manifest carries the surface; that document carries the promise. + * surface — it is a commitment, recorded with the contract's owner rather than + * here. This manifest carries the surface; the promise lives with the contract. * * Usage: node scripts/emit-contract-manifest.mjs [--check] * --check exits non-zero when the committed manifest is stale. @@ -68,7 +68,6 @@ const servedOnIndex = accepted.filter((op) => !refusedByIndex.includes(op)) const manifest = { contractVersion: versionModule.contractVersion(), engine: '@soulcraftlabs/brainy', - prose: 'docs/contract-1-ratification.md', compatibility: { minor: 'additive — a new optional door, a new served operator, a new error class; every existing implementation still conforms', From 42e2da259bcdc27b08087b3eda19e16ac77d7d65 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:27:40 -0700 Subject: [PATCH 064/150] fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gate failures on main, one real and one long-hidden. THE HEALTH-GATE PIN encoded the old law — "narrates once per generation, twice across a generation bump" — which the content-keyed dedupe deliberately replaced. A provider's `generation` bumps on every ledger mutation and every rebuild boundary, so keying narration on it re-printed an unchanged health line on every read that consulted a busy provider, and let a provider that never bumped suppress a line whose reasons had genuinely changed. The pin now asserts BOTH directions: an unchanged verdict stays silent however the counter moves, and a changed verdict is always heard. THE VFS HYBRID-SEARCH SUITE configured its store with `options.basePath`, an alias removed at the 8.0 major that configures nothing. The suite was therefore never using its temp directory — it opened the DEFAULT store, shared with every other run on the machine, and accumulated tens of thousands of rows until it failed on that shared store's graph adjacency instead of on anything it tests. It now passes `storage.path`. The suite drops from 6.5s to 0.3s, which is the measure of how much foreign data it had been opening. Neither failure was caused by the release branch; the first is the branch's own behaviour change meeting its outdated pin, the second predates it. --- tests/integration/health-gate.test.ts | 20 ++++++++++++++++---- tests/integration/hybrid-search-vfs.test.ts | 8 +++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/integration/health-gate.test.ts b/tests/integration/health-gate.test.ts index 1c9a642d..f2952116 100644 --- a/tests/integration/health-gate.test.ts +++ b/tests/integration/health-gate.test.ts @@ -188,7 +188,7 @@ describe('health gate (b) — unledgered is unknown: never blocks a serving prov describe('health gate (c) — degraded-but-serving narrates once per generation', () => { // PER-FAMILY LAW (10.4.1): a metadata find() consults the METADATA leg only — the // degraded report lives on the family the read actually consults. - it('a heal:"repair" failure serves; narrates once per generation, twice across a generation bump', async () => { + it('a heal:"repair" failure serves; narrates once per DISTINCT VERDICT, not once per generation bump', async () => { const brain = new Brainy(createTestConfig({ silent: true })) await brain.init() brains.push(brain) @@ -197,12 +197,13 @@ describe('health gate (c) — degraded-but-serving narrates once per generation' const internals = internalsOf(brain) let generation = 1 + let detail = 'counter drift' internals.metadataIndex.healthReport = () => healthReport({ provider: 'vector', serving: true, healthy: false, - invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail: 'counter drift' })], + invariants: [invariant({ name: 'stale-vector-counter', holds: false, heal: 'repair', detail })], generation }) @@ -212,11 +213,22 @@ describe('health gate (c) — degraded-but-serving narrates once per generation' await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(1) // same generation both times — one narration + expect(countNarrations()).toBe(1) // same verdict both times — one narration + // THE DEDUPE KEY IS THE VERDICT, NOT THE COUNTER. A provider's `generation` + // bumps on every ledger mutation and every rebuild boundary, so keying the + // narration on it re-printed an UNCHANGED health line on every read that + // consulted a busy provider — and, in the other direction, let a provider + // that never bumped suppress a line whose reasons had genuinely changed. + // An unchanged verdict is silent however the counter moves: generation = 2 await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) - expect(countNarrations()).toBe(2) // generation bumped — a second narration + expect(countNarrations()).toBe(1) // generation bumped, verdict identical — still silent + + // ...and a CHANGED verdict is always heard, bump or no bump: + detail = 'counter drift widened to 12 rows' + await expect(brain.find({ where: { team: 'atlas' } })).resolves.toHaveLength(1) + expect(countNarrations()).toBe(2) // the reasons changed — a new narration delete internals.metadataIndex.healthReport }) diff --git a/tests/integration/hybrid-search-vfs.test.ts b/tests/integration/hybrid-search-vfs.test.ts index c881fa97..219c4c83 100644 --- a/tests/integration/hybrid-search-vfs.test.ts +++ b/tests/integration/hybrid-search-vfs.test.ts @@ -21,10 +21,16 @@ describe('Hybrid Search with VFS', () => { testDir = path.join(os.tmpdir(), `brainy-hybrid-vfs-test-${Date.now()}`) fs.mkdirSync(testDir, { recursive: true }) + // `storage.path`, NOT the pre-8.0 `options.basePath` alias. That alias was + // removed at the 8.0 major and configures nothing, so this suite silently + // opened the DEFAULT store instead of its own temp directory — sharing one + // on-disk brain with every other run on the machine, accumulating tens of + // thousands of rows, and eventually failing on that shared store's graph + // adjacency rather than on anything it was written to test. brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', - options: { basePath: testDir } + path: testDir } }) await brain.init() From d49148e1407cc14a5de031f605e081bff30cb399 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:30:30 -0700 Subject: [PATCH 065/150] fix(vfs): the old-root sweep narrates only when it has something to say MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release gate's own output caught this: every test brain printed "[VFS] old-root sweep complete in 1ms and recorded" — hundreds of lines — and a consumer would get two of them on the first open of every store. They were emitted on the always-visible channel, which a production log level deliberately CANNOT silence. That channel exists so an operator can always learn why a database is slow; a 0ms no-op on a fresh store is not that, and announcing it there trains people to ignore the one channel built to be impossible to ignore. It was also inconsistent with every other narration in this work, all of which is silent under a threshold. The sweep now speaks when it has something to say — duplicate roots removed, or a wall over a second that a person watching a slow first open deserves explained — and otherwise does its work, records its marker, and stays quiet. cleanupOldRoots() reports what it removed so the decision rests on a fact rather than on a guess. Pin: a fresh store's sweep emits nothing on the channel and still records its marker, so the silence can never be mistaken for the work being skipped. --- src/vfs/VirtualFileSystem.ts | 40 ++++++++++++++----- tests/integration/vfs-root-sweep-once.test.ts | 27 +++++++++++++ 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 19470b48..46c6a12d 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -76,6 +76,11 @@ export class VirtualFileSystem implements IVirtualFileSystem { * `_system/`, like every other marker there — never enumerated as data. */ private static readonly ROOT_SWEEP_MARKER_PATH = '_system/vfs-root-sweep.json' + /** + * Below this wall, a sweep that removed nothing says nothing — see + * {@link sweepOldRootsIfNeeded}. + */ + private static readonly ROOT_SWEEP_NARRATE_MS = 1_000 private currentUser: string = 'system' // Track current user for collaboration // Knowledge Layer features available via augmentation (brain.use('knowledge')) @@ -434,21 +439,31 @@ export class VirtualFileSystem implements IVirtualFileSystem { } catch { // Unreadable marker: sweep, and rewrite it below. } - prodLog.narrate( - '[VFS] one-time sweep for pre-fixed-id root directories running in the background — ' + - 'the open does not wait for it, and once it has run this store never sweeps again.' - ) + // NARRATION HAS A THRESHOLD, like every other line this engine emits on the + // always-visible channel. On a fresh or small store this sweep finds + // nothing and costs a millisecond, and announcing it — twice — on a + // channel a production log level deliberately CANNOT silence would train + // operators to ignore the one channel that exists to be impossible to + // ignore. It speaks when it has something to say: duplicates removed, or a + // wall long enough that somebody watching a slow first open deserves to + // know what is running. Otherwise it does its work and stays quiet. const startedAt = Date.now() - await this.cleanupOldRoots() + const duplicatesRemoved = await this.cleanupOldRoots() + const elapsedMs = Date.now() - startedAt try { await store.writeRawObject(VirtualFileSystem.ROOT_SWEEP_MARKER_PATH, { sweptAt: new Date().toISOString(), - durationMs: Date.now() - startedAt + durationMs: elapsedMs }) - prodLog.narrate( - `[VFS] old-root sweep complete in ${Date.now() - startedAt}ms and recorded — ` + - 'no future open pays for it.' - ) + if (duplicatesRemoved > 0 || elapsedMs >= VirtualFileSystem.ROOT_SWEEP_NARRATE_MS) { + prodLog.narrate( + `[VFS] one-time old-root sweep complete in ${elapsedMs}ms` + + (duplicatesRemoved > 0 + ? `, ${duplicatesRemoved} pre-fixed-id root(s) removed` + : '') + + ' and recorded — no future open pays for it.' + ) + } } catch (error) { // Unrecorded sweep = the next open sweeps again. Conservative, and said // out loud rather than quietly repeated forever. @@ -495,7 +510,8 @@ export class VirtualFileSystem implements IVirtualFileSystem { return null } - private async cleanupOldRoots(): Promise { + private async cleanupOldRoots(): Promise { + let removed = 0 try { // Find any old VFS roots with UUID-based IDs (not our fixed ID) const oldRoots = await this.brain.find({ @@ -517,6 +533,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { for (const duplicate of duplicates) { try { await this.brain.remove(duplicate.id) + removed++ console.log(`VFS: Deleted old root ${duplicate.id.substring(0, 8)}`) } catch (error) { console.warn(`VFS: Failed to delete old root ${duplicate.id}:`, error) @@ -529,6 +546,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Non-critical error - log and continue console.warn('VFS: Cleanup of old roots failed (non-critical):', error) } + return removed } /** diff --git a/tests/integration/vfs-root-sweep-once.test.ts b/tests/integration/vfs-root-sweep-once.test.ts index f84f4413..cac59b70 100644 --- a/tests/integration/vfs-root-sweep-once.test.ts +++ b/tests/integration/vfs-root-sweep-once.test.ts @@ -20,6 +20,7 @@ import { join } from 'node:path' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' +import { prodLog } from '../../src/utils/logger.js' describe('the VFS old-root sweep', () => { const dirs: string[] = [] @@ -80,6 +81,32 @@ describe('the VFS old-root sweep', () => { expect(sweepSpy).not.toHaveBeenCalled() }, 180_000) + it('a sweep that removes nothing on a fresh store says nothing', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-quiet-')) + dirs.push(dir) + + // The always-visible channel cannot be silenced by a log level, so a line + // on it has to earn its place. A fresh store's sweep finds no duplicate + // roots and costs a millisecond — it must do its work, record its marker, + // and stay quiet, or it trains operators to ignore the one channel that + // exists to be impossible to ignore. + const narrated: string[] = [] + const spy = vi.spyOn(prodLog, 'narrate').mockImplementation(((...args: unknown[]) => { + narrated.push(args.map((a) => String(a)).join(' ')) + }) as typeof prodLog.narrate) + + const brain = await open(dir) + await (brain.vfs as unknown as { whenRootSweepSettled: () => Promise }).whenRootSweepSettled() + spy.mockRestore() + + expect(narrated.filter((l) => /old-root sweep/i.test(l))).toEqual([]) + // ...and it still did the work: the marker is recorded, so no future open sweeps. + expect( + existsSync(join(dir, '_system', 'vfs-root-sweep.json')) || + existsSync(join(dir, '_system', 'vfs-root-sweep.json.gz')) + ).toBe(true) + }, 180_000) + it('the open does not wait for the sweep', async () => { const dir = mkdtempSync(join(tmpdir(), 'brainy-root-sweep-async-')) dirs.push(dir) From ff39941b0a8795d0dd9fd99a19395f6a9708cebb Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 12:39:55 -0700 Subject: [PATCH 066/150] chore(release): 10.4.4 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7790837..a54d609e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,35 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) + +- fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) +- fix(tests): the health-gate pin follows the verdict, and the VFS suite uses its own store (42e2da25) +- Merge branch 'next/open-lazy-open-and-counts' (5ebd3b40) +- docs: the contract manifest stands alone; public docs describe this engine only (a8c724a2) +- docs(releases): 10.4.4 consumer notes — correctness and observability, with the performance line stated exactly (61a46927) +- docs: measurements in public history carry numbers, not provenance (02c61636) +- feat(open): name the two steps that hold the vfs-bootstrap phase (2cf38010) +- fix(storage): a dead flush watch falls back to the 500ms poll, not the 30s sweep (5c22f950) +- fix(storage): the flush watcher cannot arm twice in its async window (16d2e1a9) +- perf(idle): the flush-request watch is event-driven; the heartbeat is observability (fb1da1c5) +- perf(open): answer "are there any entities?" with one directory read (417ddb51) +- perf(generations): discover generations by directory name, not by walking the log (9dd39921) +- fix(flush): clear() and repairIndex() set the dirty witness themselves (e4c27fbc) +- feat(open): the open names the STEP that cost the time, not just the phase (5a091cca) +- perf(vfs): the old-root sweep runs once per store, not once per open (4a67aa0f) +- chore: keep the generated neural stamps at main's values (c1f09723) +- feat(contract): declare contract 1, serve three operators, refuse four by name (48802ba3) +- fix(open): a provider rebuilding itself is a third state, not a CRITICAL (50676c02) +- feat(open): open never waits for a provider that is rebuilding itself (131daa08) +- perf(flush): an idle brain does no work — no periodic flush without a write (f5a6cb3f) +- feat(repair): repairIndex narrates every phase and its receipt carries the walls (3fffd9c6) +- fix(storage): a suspect count ledger heals itself, and counts.json is written atomically (f4e2d34b) +- feat(open): the open narrates itself, on a channel production cannot clamp (afe08a1f) +- fix(storage): a clean close is recorded, and the writer lock is always given up (e652162c) +- docs: repository links point at soulcraftlabs/open-brainy — the soulcraft/brainy path becomes the native engine's repo tonight (38c3397b) + + ### [10.4.3](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.2...v10.4.3) (2026-08-27) - Merge branch 'next/open-brainy-rename' (a58372f0) diff --git a/package-lock.json b/package-lock.json index 4d247780..c4f66561 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.3", + "version": "10.4.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.3", + "version": "10.4.4", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 60e901de..06ce0253 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.3", + "version": "10.4.4", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From b8475cc86a9c5dca8c5c34f84f1e314e76369ef7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 28 Aug 2026 13:00:42 -0700 Subject: [PATCH 067/150] =?UTF-8?q?fix(release):=20the=20release=20page=20?= =?UTF-8?q?posts=20to=20this=20repository=20=E2=80=94=20soulcraftlabs/open?= =?UTF-8?q?-brainy,=20never=20the=20engine's?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 11 POSTed to repos/soulcraft/brainy while printing the correct URL; dormant only because FORGEJO_RELEASE_TOKEN was unset. Found during the 10.4.4 cut verification. --- scripts/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.sh b/scripts/release.sh index 08293e3a..67ad1053 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -237,7 +237,7 @@ fi # and RELEASES.md are the record; this just gives The Source's UI a release page). echo -e "${BLUE}🔟 Creating release page on The Source...${NC}" if [ -n "${FORGEJO_RELEASE_TOKEN:-}" ]; then - if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraft/brainy/releases" \ + if curl -sf -X POST "https://source.soulcraft.com/api/v1/repos/soulcraftlabs/open-brainy/releases" \ -H "Authorization: token ${FORGEJO_RELEASE_TOKEN}" -H "Content-Type: application/json" \ -d "{\"tag_name\":\"v${NEW_VERSION}\",\"name\":\"v${NEW_VERSION}\",\"prerelease\":${PRERELEASE}}" >/dev/null; then echo -e "${GREEN}✅ Release page created on The Source${NC}\n" From 298cb6dacaac9ef80db65a723d65cbd53c15d23e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:07:18 -0700 Subject: [PATCH 068/150] fix(recovery): a torn generation-log tail is a terminal verdict, never a wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of one defect, found by a seeded-SIGKILL crash lane. THE FALSE POSITIVE. stampEntityTree() recorded generationStore.generation() — the ALLOCATED counter, a number a write in flight has claimed and may never commit — while the JSDoc beside it already said the source is the committed generation. Every crash inside a write window therefore produced a spurious verdict at the next open: either 'sourceGeneration N is ahead of the log head N-1' (the allocated generation died with the process) or 'rollup invariant nounCount: stamped X, observed Y' (the recovery fold folded facts the stamp's counts predate). Both told the operator to run repairIndex() — a whole-store recount — for a store that was coherent. Measured before this commit: 4 of 11 SIGKILL cycles on a healthy store raised one of the two. The stamp and the open now both read committedGeneration(), which is what every other open-time watermark in the class already reasons about. THE TERMINAL VERDICT. A stamp still ahead of committed truth after the recovery fold witnesses a generation that is not in the log — the stamp's fsync outlived the tail's, and there is nothing to arrive. That is its own verdict state now ('torn'), never folded in with 'incoherent': the two have opposite cures. A writer open demotes it — the unusable stamped surface is re-derived at the committed generation from the live counters, O(1), straight-line, no loop and no await on external progress, narrated with both count sets, the stamp's path and its committedAt. A read-only open cannot re-stamp, so it says so and names the cure instead of guessing, and still serves. Neither branch waits, and neither locks an owner out of a canonical tree the stamp only describes. Pins: the verifier returns the torn verdict with both generations; a fabricated head-behind-source store narrates precisely, demotes inside a bounded open, serves its rows, and is quiet at the next open (the demotion converges); a read-only open narrates the same verdict and leaves the bytes untouched. --- src/brainy.ts | 121 +++++++++++++++++++- src/db/familyStamp.ts | 32 ++++-- tests/integration/entity-tree-stamp.test.ts | 104 ++++++++++++++++- 3 files changed, 241 insertions(+), 16 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 70d46973..da04577e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -12497,6 +12497,18 @@ export class Brainy implements BrainyInterface { * healed by `repairIndex()`, whose unconditional recount rebuilds the * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write * fault warns loudly but never fails the flush that carried real data. + * + * THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is + * the ALLOCATED counter — a number a write in flight has claimed and may + * never commit. Stamping it made the stamp's generation label a claim about + * counts it was not taken at, and every crash inside a write window then + * produced a spurious verdict at the next open: either `sourceGeneration N + * is ahead of the log head N-1` (the allocated generation died with the + * process) or `rollup invariant 'nounCount': stamped X, observed Y` (the + * recovery fold folded facts the stamp's counts predate). MEASURED on the + * crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on + * a coherent store raised one of those two verdicts, each of them naming + * `repairIndex()` — a whole-store recount — as the cure for nothing. */ private async stampEntityTree(): Promise { if (this.isReadOnly) return @@ -12507,7 +12519,7 @@ export class Brainy implements BrainyInterface { ]) await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { family: 'entity-tree', - sourceGeneration: this.generationStore.generation(), + sourceGeneration: this.generationStore.committedGeneration(), members: { mode: 'rollup', invariants: { nounCount, verbCount } } }) } catch (error) { @@ -12520,16 +12532,24 @@ export class Brainy implements BrainyInterface { /** * @description Open-time coherence check for the entity tree's family stamp: - * compare `sourceGeneration` against the log head and the stamped rollup - * invariants against the live counters. Verdicts: + * compare `sourceGeneration` against the store's COMMITTED generation and + * the stamped rollup invariants against the live counters. Verdicts: * - `coherent` / `absent` (legacy store; first flush stamps) → silent. * - `behind` → benign for the tree (it is written BY the commit; only the * stamp is stale — a crash landed between commit and flush). Refreshed at * the next flush. + * - `torn` → a TORN GENERATION-LOG TAIL, handled by + * {@link demoteTornEntityTreeStamp}: terminal, never a wait. * - `incoherent` → LOUD: the tree or its counters diverged from what was * stamped — `repairIndex()` recounts from canonical and re-stamps. * Never blocks open; a fault reading the stamp is surfaced as unverifiable, * never conflated with absence. + * + * THE COMPARISON IS AGAINST `committedGeneration()`, matching what + * {@link stampEntityTree} writes and what every other open-time watermark in + * this class already reasons about (the fact-scan capability, the metadata / + * graph / HNSW watermark verdicts). Comparing against the allocated counter + * was the one place that disagreed, and disagreeing was the whole defect. */ private async verifyEntityTreeStamp(): Promise { let stamp: FamilyStamp | null @@ -12546,11 +12566,16 @@ export class Brainy implements BrainyInterface { this.storage.getNounCount(), this.storage.getVerbCount() ]) - const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { + const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), { nounCount, verbCount }) - if (verdict.state === 'incoherent') { + if (verdict.state === 'torn') { + await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, { + nounCount, + verbCount + }) + } else if (verdict.state === 'incoherent') { prodLog.warn( `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + `The canonical tree or its counters diverged from the stamped state — run ` + @@ -12564,6 +12589,92 @@ export class Brainy implements BrainyInterface { } } + /** + * @description THE TERMINAL VERDICT for a torn generation-log tail. + * + * A stamp whose `sourceGeneration` sits ABOVE the store's committed + * watermark witnesses a generation that is not in the log: the stamp's fsync + * outlived the tail's. By the time this runs, log-authority recovery has + * already folded every intact fact above the manifest and advanced the + * watermark to cover them — so if the stamp is STILL ahead, the generation + * it names is not merely late, it is GONE. There is nothing to wait for. + * + * That is the whole point of this method. A field report of this class + * (single-process store, abrupt termination mid-fold) described a reopen + * that narrated the tear and then held 100% CPU with zero log growth for + * eight minutes before an operator wiped the directory. A recovery that + * cannot say what it is waiting for has no business spinning; the honest + * answer here is a verdict, taken now, at O(1) cost. + * + * WHAT THE VERDICT DOES — the stamped surface is UNUSABLE, so it is + * discarded rather than believed: the stamped counts describe a generation + * that never became durable, and comparing them against live counters can + * only produce noise. The tree itself is not in question (it IS canonical — + * every commit writes it, and the fold re-applied every after-image the log + * still holds), so the demotion is a re-derivation of this family's verified + * surface at the generation the store can actually show: + * + * - WRITER open → re-stamp at `committedGeneration()` from the live + * counters — exactly what the next flush would write, taken now so the + * tear cannot re-narrate on every subsequent open. Both count sets are + * logged so an operator can see whether anything really moved. + * - READER open → a reader cannot re-stamp. Narrate the same terminal + * verdict with the named cure and carry on serving; a read-only inspector + * is never locked out of a store, and never left waiting either. + * + * BOUNDEDNESS: straight-line code. No loop, no retry, no await on any + * external progress signal — the two counter reads and one stamp write are + * the entire cost, and none of them scales with the store. + */ + private async demoteTornEntityTreeStamp( + stamp: FamilyStamp, + stampSource: number, + head: number, + observed: { nounCount: number; verbCount: number } + ): Promise { + const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {} + const detail = + `[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` + + `generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` + + `generation is ${head} after crash recovery — the stamp's fsync outlived the log tail's, ` + + `and generation ${stampSource} is not in the log to arrive. Stamped rollups ` + + `${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.` + + if (this.isReadOnly) { + prodLog.warn( + `${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` + + `family stays UNVERIFIED for this session (reads are unaffected — the canonical tree ` + + `is the truth this stamp only describes). Cure: open the store with a writer, or run ` + + `brain.repairIndex() there, to recount from canonical and re-stamp.` + ) + return + } + + const startedAt = Date.now() + try { + await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { + family: 'entity-tree', + sourceGeneration: head, + members: { + mode: 'rollup', + invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount } + } + }) + prodLog.warn( + `${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` + + `from the live counters in ${Date.now() - startedAt}ms — terminal, not a wait. If the ` + + `observed counts above look wrong for your data, run brain.repairIndex() to recount ` + + `from canonical.` + ) + } catch (error) { + prodLog.warn( + `${detail} The demotion's re-stamp FAILED (${(error as Error).message}) — the tear will ` + + `narrate again at the next open, which is the honest outcome; the store still serves ` + + `from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.` + ) + } + } + /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts index 98342884..2f01e935 100644 --- a/src/db/familyStamp.ts +++ b/src/db/familyStamp.ts @@ -12,9 +12,11 @@ * the verified surface is a small set of rollup invariants (entity/ * relationship counts) plus `sourceGeneration`. * - * `sourceGeneration` is the generation of the source-of-truth log this - * projection reflects — open-time coherence becomes a COMPARISON (stamp vs - * log head), not a walk: + * `sourceGeneration` is the COMMITTED generation of the source-of-truth log + * this projection reflects — never the allocated counter, which names a + * generation that may never commit (see {@link StampVerdict.torn}) — so + * open-time coherence becomes a COMPARISON (stamp vs committed head), not a + * walk: * * - equal + invariants hold → coherent, serve. * - behind → the projection missed the tail (crash between commit and stamp); @@ -24,6 +26,9 @@ * - invariants FAIL at equal generation → genuine incoherence: loud, and the * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * canonical walk) heals it. + * - AHEAD → a torn generation-log tail: the stamp's fsync outlived the log + * tail's. TERMINAL, never a wait — the generation the stamp names does not + * exist to arrive. * * Stamps are JSON on purpose — every incident gets debugged by reading a * stamp in a terminal. @@ -70,6 +75,12 @@ export type StampVerdict = | { state: 'coherent' } | { state: 'absent' } // legacy store — first stamp writes at the next flush | { state: 'behind'; stampSource: number; head: number } + /** + * TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the + * store's committed watermark can no longer show. TERMINAL — there is no + * generation to wait for, so the open demotes (or refuses) and never spins. + */ + | { state: 'torn'; stampSource: number; head: number } | { state: 'incoherent'; failures: string[] } | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence @@ -118,12 +129,15 @@ export function verifyFamilyStamp( ): StampVerdict { if (stamp === null) return { state: 'absent' } if (stamp.sourceGeneration > head) { - // A stamp AHEAD of the log claims state that never committed — the - // projection was stamped against truth that a crash rolled back. - return { - state: 'incoherent', - failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] - } + // A stamp AHEAD of committed truth witnesses a generation the store can no + // longer show: the stamp's fsync survived a crash that the log tail did + // not. This is the TORN GENERATION-LOG TAIL — its own class, never folded + // in with `incoherent` (a count that drifted at a generation both sides + // agree on), because the two have opposite cures: incoherence is recounted, + // a tear is DEMOTED. It is also terminal by construction — there is no + // generation the open can wait for, because the one the stamp names is + // gone. + return { state: 'torn', stampSource: stamp.sourceGeneration, head } } if (stamp.sourceGeneration < head) { return { state: 'behind', stampSource: stamp.sourceGeneration, head } diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts index deefc5e6..23cc0a15 100644 --- a/tests/integration/entity-tree-stamp.test.ts +++ b/tests/integration/entity-tree-stamp.test.ts @@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => { const invariants = (stamp.members as any).invariants expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) - expect(stamp.sourceGeneration).toBe(brain.generation()) + // THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the + // counter labelled the stamp with a generation a write in flight had merely + // claimed, so every crash inside a write window produced a spurious verdict + // at the next open (see the torn-tail pins below). + expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration()) expect(stamp.generation).toBeGreaterThanOrEqual(1) }) @@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => { expect(stillIncoherent).toEqual([]) }) + /** + * Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's + * committed watermark — the durable shape a torn generation-log tail leaves + * behind (the stamp's fsync outlived the tail's). Fabricated rather than + * crash-produced so the pin is deterministic; the seeded-SIGKILL lane + * (`scripts/crash-consistency.mjs` in the engine repo) produces the same + * shape from a real abrupt termination. + */ + const fabricateTear = (ahead: number): FamilyStamp => { + const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`) + const zlib = require('node:zlib') + const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp + const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead } + fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn))) + return torn + } + + it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => { + for (let i = 0; i < 3; i++) + await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } }) + await brain.close() + const torn = fabricateTear(5) + + const warn = vi.spyOn(prodLog, 'warn') + const startedAt = Date.now() + brain = await open() + const openMs = Date.now() - startedAt + + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + // Narrated PRECISELY: both generations, the file, and the named cure. + expect(said).toContain(`source generation ${torn.sourceGeneration}`) + expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`) + expect(said).toContain(ENTITY_TREE_STAMP_PATH) + expect(said).toContain('DEMOTED') + expect(said).toMatch(/repairIndex\(\)/) + // Terminal, not a wait: the demotion is O(1) straight-line work, so a tear + // cannot turn an open into the 8-minute spin this class was reported as. + expect(openMs).toBeLessThan(30_000) + + // The store SERVES — a tear in a stamp never locks an owner out of the + // canonical tree the stamp merely describes. + expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3) + + // The demotion CONVERGED: the stamp now names committed truth, and the + // next open is quiet. A verdict that re-narrates every open is a wait + // wearing a different hat. + const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp + expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration()) + await brain.close() + const warn2 = vi.spyOn(prodLog, 'warn') + brain = await open() + expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([]) + }) + + it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => { + await brain.add({ data: 'ro', type: 'document', metadata: {} }) + await brain.close() + const torn = fabricateTear(3) + + const warn = vi.spyOn(prodLog, 'warn') + const reader: any = await Brainy.openReadOnly({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + expect(said).toContain('READ-ONLY') + expect(said).toContain('UNVERIFIED') + expect(said).toMatch(/repairIndex\(\)/) + await reader.close() + + // A reader never rewrites the store: read the bytes back off disk (not + // through a writer open, which would demote them) — the torn stamp is + // exactly as it was found. + const onDisk = JSON.parse( + require('node:zlib') + .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) + .toString('utf-8') + ) as FamilyStamp + expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration) + expect(onDisk.generation).toBe(torn.generation) + + brain = await open() + }) + it('the one verifier handles both member modes', () => { const rollup: FamilyStamp = { family: 'x', @@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => { stampSource: 5, head: 9 }) - expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head + // AHEAD is its own class — a torn generation-log tail, never folded in + // with `incoherent`: the two have opposite cures (recount vs demote). + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({ + state: 'torn', + stampSource: 5, + head: 3 + }) expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) const enumerated: FamilyStamp = { From 9a888c37e9ebec5573cd7ebd0764396f3a424de3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:13:42 -0700 Subject: [PATCH 069/150] fix(generations): a sealed segment may only declare the generations it holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis of the "packed history is damaged" narration that fires on every run of the affected stores. It is a WRITER defect, and the reader's refusal was the symptom rather than the cause. A sealed segment declares one contiguous range [firstGeneration, lastGeneration], and every reader treats that range as containment: coveringSegment is an interval test, hasGeneration returns true for anything inside it, and open() seeds committedRanges from it. repackHistory handed fold() a SPARSE batch. Three filters punch holes in its candidate list mid-run — a generation absent from committedRanges never appears, one still in the pending buffer is skipped, one whose tx.json will not read is skipped — and fold() then computed the range from the first and last survivor, claiming every generation in between. The next open merged that mis-declared range back into committedRanges, re-admitting the hole as committed history, so the following auto-compaction pass asked the packed tier for a frame that was never written and failed. Re-merged at every open, which is why it repeated on every run. Confirmed against a forensic fixture: generation directories 1..2503 present except exactly one, 1416; and its fact-log segment already showed the tell — seg-...1410.bfl declaring 1410..1940 (531 generations) while recording 530 facts. Three changes: - repackHistory folds each contiguous RUN as its own segment (`contiguousRuns`), so ranges describe exactly what the segments contain. - fold() REFUSES a non-contiguous batch, naming the gap and its width. The density law is now mechanical, so no future caller can reintroduce it. A refusal loses nothing: the generations stay live and readable. - Stores already carrying the damage heal instead of wedging. A segment whose declared span exceeds its frame count is SPARSE; `actualRanges()` reads the real generation list from its sidecar so open() never re-admits the holes, and readFrame reports such a hole as unpacked with a narration naming the segment, rather than throwing. A DENSE segment missing a frame is still loud damage — that one means the manifest and sidecar disagree. Pins: nine unit cases (refusal and its message, honest ranges for separately folded runs, a reconstructed pre-fix sparse segment serving its real frames while reporting holes as unpacked, holes excluded from actualRanges, and the dense-segment damage path still throwing) plus an end-to-end case that deletes a generation directory and drives the real sequence — ordinary close()-time repacking folds over the hole, then reopen and compact must both complete. Verified red without the fix: the segment declared an 11-generation span while holding 10 frames. --- src/db/generationSegments.ts | 119 +++++++++++++++++++- src/db/generationStore.ts | 66 +++++++++-- tests/integration/history-repacking.test.ts | 102 +++++++++++++++++ tests/unit/db/generation-segments.test.ts | 115 +++++++++++++++++++ 4 files changed, 389 insertions(+), 13 deletions(-) diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts index 0c14b60c..91451281 100644 --- a/src/db/generationSegments.ts +++ b/src/db/generationSegments.ts @@ -147,6 +147,60 @@ export class GenerationSegmentStore { return this.coveringSegment(gen) !== null } + /** + * @description True when `meta` declares more generations than it holds + * frames — a segment sealed by a writer that folded across a hole. The + * manifest records `frames` at fold time, so this is an O(1) comparison + * against the declared span and needs no I/O. + */ + private isSparse(meta: SegmentMeta): boolean { + return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames + } + + /** + * @description The generations this tier ACTUALLY holds, as coalesced + * ascending intervals — not what the segments declare. + * + * Dense segments (every one a current writer produces) contribute their + * declared range with no I/O. A SPARSE segment — one sealed before the + * density law was enforced, whose declared range spans generations it has + * no frame for — has its real generation list read from its sidecar and + * contributed instead, with the discrepancy narrated once. + * + * This is what keeps a store that already carries the damage from wedging. + * `open()` seeds `committedRanges` from these intervals, so a hole is never + * re-admitted as a committed generation, and the auto-compaction pass that + * used to fail on every run with "packed history is damaged" simply never + * asks for the missing frame. + * + * @returns Ascending, non-overlapping `[first, last]` intervals. + */ + async actualRanges(): Promise> { + const out: Array<[number, number]> = [] + for (const meta of this.manifest.segments) { + if (!this.isSparse(meta)) { + out.push([meta.firstGeneration, meta.lastGeneration]) + continue + } + const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames + prodLog.warn( + `[GenerationSegments] sealed segment ${meta.file} declares generations ` + + `${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` + + `frame(s) — ${missing} generation(s) in that span were never folded into it. ` + + `Serving the frames it actually holds; the declared span is not treated as ` + + `committed history. (Written by a pre-density-law writer that folded across a ` + + `gap; the segment itself is intact and no record is lost.)` + ) + const idx = await this.sidecarFor(meta) + for (const [gen] of idx.generations) { + const last = out[out.length - 1] + if (last !== undefined && gen === last[1] + 1) last[1] = gen + else out.push([gen, gen]) + } + } + return out + } + /** * Fold consecutive generations into ONE new sealed segment + sidecar and * append it to the manifest atomically. Caller guarantees: `gens` is @@ -164,6 +218,38 @@ export class GenerationSegmentStore { throw new Error('[GenerationSegments] fold() input must be strictly ascending') } } + // THE DENSITY LAW, MADE MECHANICAL. + // + // A sealed segment declares a CONTIGUOUS range [firstGeneration, + // lastGeneration] and every reader treats that range as containment: + // `coveringSegment` is an interval test, `hasGeneration` returns true for + // anything inside it, and `open()` seeds committedRanges from it. So a + // segment folded from a SPARSE input silently claims generations it does + // not hold, and the first read of one of those holes throws + // "inside sealed segment ... but has no frame — packed history is damaged". + // + // That is exactly how the damage was produced. `repackHistory` skipped + // generations mid-batch — ones absent from committedRanges, ones still in + // the pending buffer, ones whose tx.json would not read — and handed the + // survivors here, where the range was computed from the first and last of + // them. Worse, the mis-declared range was then merged back into + // committedRanges at the next open, which is what turned a quiet hole into + // a repeating auto-compaction failure on every subsequent run. + // + // Callers now split at discontinuities; this refusal is what keeps any + // future caller from reintroducing the class. A refusal here loses + // nothing — the generations stay in the live tier, readable, and the next + // pass folds them correctly. + for (let i = 1; i < gens.length; i++) { + if (gens[i].generation !== gens[i - 1].generation + 1) { + throw new Error( + `[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` + + `${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` + + `generation(s). A sealed segment declares a dense range, so folding a sparse ` + + `batch would claim generations it does not hold. Split the batch at the gap.` + ) + } + } const last = this.manifest.segments[this.manifest.segments.length - 1] if (last && gens[0].generation <= last.lastGeneration) { throw new Error( @@ -364,12 +450,37 @@ export class GenerationSegmentStore { return this.decodeFrame(payload) } } - // In the covering range but not present: the packed tier is dense by - // construction (fold packs every generation it is handed, including - // record-less ones) — absence inside a sealed range is damage. + // Inside the covering range but with no frame. Two very different causes, + // and conflating them is what made this class wedge every maintenance pass + // on the affected stores. + // + // (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than + // the span it declares. That segment was sealed by a writer that + // folded across a hole (the class this file's density law now bars). + // The segment is INTACT and nothing is lost; it simply never held this + // generation. Answering "not packed" is the honest answer, and it lets + // the caller's two-tier read decide what a genuinely absent generation + // means, instead of every compaction pass dying on a repeating throw. + // `actualRanges()` keeps such holes out of committedRanges at open, so + // in a healed store nobody asks this question in the first place. + // + // (2) A DENSE SEGMENT missing a frame it says it has — the manifest and + // the sidecar disagree about a segment that claims to be complete. + // That IS damage, and it stays loud. + if (this.isSparse(meta)) { + prodLog.warn( + `[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` + + `declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` + + `holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` + + `-generation span — it was sealed across a gap and never held this generation. ` + + `Reporting it as unpacked rather than as damage; no record is lost.` + ) + return null + } throw new Error( `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame — packed history is damaged` + `range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + + `span — the manifest and the sidecar disagree; packed history is damaged` ) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index fd052c31..da21dc61 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -96,6 +96,35 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' +/** + * @description Split an ascending list of fold candidates into maximal + * CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`. + * + * A sealed segment declares one dense range `[firstGeneration, + * lastGeneration]`, and every reader treats that range as containment. So a + * batch with a hole in it must never become one segment: it would claim a + * generation it does not hold, and the first read of that hole reports the + * packed history as damaged. One run, one segment — the ranges then describe + * exactly what the segments contain. + * + * @param gens - Fold candidates, strictly ascending by generation. + * @returns One array per contiguous run, in ascending order. Empty in, empty out. + */ +export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] { + const runs: FoldGeneration[][] = [] + let run: FoldGeneration[] = [] + for (const g of gens) { + const prev = run[run.length - 1] + if (prev !== undefined && g.generation !== prev.generation + 1) { + runs.push(run) + run = [] + } + run.push(g) + } + if (run.length > 0) runs.push(run) + return runs +} + /** * @description Phases of the {@link GenerationStore.commitTransaction} commit * protocol at which a test-only fault injector can simulate a process crash. @@ -784,9 +813,15 @@ export class GenerationStore { if (storageSupportsFactLog(this.storage)) { this.segments = new GenerationSegmentStore(this.storage) await this.segments.open() - const packedRanges = this.segments - .segments() - .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) + // ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law + // writer can declare a span wider than the frames it holds; seeding + // committedRanges from the declared span re-admits those holes as + // committed generations, and every later maintenance pass then asks for a + // frame that was never written. `actualRanges()` reads the real + // generation list from the sidecar for exactly those segments (and does + // no I/O for the dense ones, which is all of them on a healthy store). + const packedRanges = (await this.segments.actualRanges()) + .map((r): [number, number] => [r[0], Math.min(r[1], this.committed)]) .filter(([lo, hi]) => lo <= hi) if (packedRanges.length > 0) { // Merge packed (older) + live (newer) interval sets — both ascending; @@ -3121,13 +3156,26 @@ export class GenerationStore { foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) } if (foldInput.length === 0) continue - await segments.fold(foldInput) - segmentsCreated++ - // Segment + manifest durable → the live copies retire. - for (const g of foldInput) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + // SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three + // filters above punch holes in it: a generation missing from + // committedRanges never appears, one still in the pending buffer is + // skipped, and one whose tx.json will not read is skipped. A sealed + // segment declares a DENSE range, so folding across such a hole makes + // the segment claim a generation it does not hold; the next open + // merges that mis-declared range into committedRanges, and every + // subsequent auto-compaction pass then asks for the missing frame and + // fails with "packed history is damaged". Fold each contiguous RUN as + // its own segment instead — same bytes, honest ranges. + for (const run of contiguousRuns(foldInput)) { + if (deadline !== undefined && Date.now() >= deadline) break + await segments.fold(run) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of run) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + } + folded += run.length } - folded += foldInput.length } if (folded > 0) { prodLog.info( diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts index 2bcee038..bb07268d 100644 --- a/tests/integration/history-repacking.test.ts +++ b/tests/integration/history-repacking.test.ts @@ -16,6 +16,7 @@ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as path from 'node:path' import * as os from 'node:os' +import * as zlib from 'node:zlib' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { GenerationStore } from '../../src/db/generationStore.js' @@ -57,6 +58,107 @@ describe('history repacking — the two-tier lifecycle', () => { } }) + /** + * THE HOLE, END TO END — the shape a real store carries. + * + * A forensic fixture was measured with generation directories 1..2503 + * present except for exactly one: 1416. Its fact-log segment already showed + * the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration + * 1940 (531 generations) while recording only 530 facts. + * + * Before the fix, repacking such a store folded ACROSS that hole: the batch + * skipped 1416 (no readable delta) and the sealed segment declared a range + * spanning it anyway. The next open merged that declared range back into + * committedRanges, re-admitting 1416 as committed history, and every + * subsequent auto-compaction pass then asked the packed tier for a frame + * that was never written — producing, on EVERY run, the non-fatal narration + * + * Auto-compaction of generational history failed (non-fatal): generation + * N is inside sealed segment seg-....bgs's declared range but has no frame + * — packed history is damaged + * + * This pin removes a generation directory to make the same hole, then + * requires repack + reopen + compaction to complete cleanly. + */ + it('a missing generation directory does not poison the packed tier', async () => { + const dir = tempDir() + // `retention: 'all'` throughout: close() otherwise auto-compacts the + // history away, and this pin needs the cold generations still on disk so + // there is something to punch a hole in. The live window stays at its + // production default for the build phase, so nothing folds yet. + const archival = async (): Promise => { + const b = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + embeddingFunction: stub, + retention: 'all' + }) + await b.init() + return b + } + const brain = await archival() + + const id = await brain.add({ + data: 'holed-entity', + type: NounType.Document, + metadata: { v: 0 } + }) + // One flush per update: single-op writes coalesce inside a flush window, + // so a history deep enough to have a middle needs the windows separated. + for (let v = 1; v <= 12; v++) { + await brain.update({ id, metadata: { v } }) + await brain.flush() + } + await brain.close() + + // Punch the hole: delete ONE generation directory in the middle of the + // cold range, exactly as the real store presents it. + const genRoot = path.join(dir, '_generations') + const numeric = fs + .readdirSync(genRoot, { withFileTypes: true }) + .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)) + .map((e) => Number(e.name)) + .sort((a, b) => a - b) + expect(numeric.length).toBeGreaterThan(6) + const victim = numeric[Math.floor(numeric.length / 2)] + fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true }) + + // Now shrink the live window and reopen. close() repacks automatically + // (brainy.ts phase 0b), so this is the production sequence exactly: a + // store with a hole in its history gets folded by ordinary housekeeping, + // with nobody asking for it. + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 + const reopened = await archival() + const result = await reopened.repackHistory() + expect(result.foldedGenerations).toBeGreaterThan(0) + + const segDir = path.join(dir, SEGMENTS_PREFIX) + const manifestPath = ['manifest.json', 'manifest.json.gz'] + .map((f) => path.join(segDir, f)) + .find((p) => fs.existsSync(p))! + const raw = manifestPath.endsWith('.gz') + ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8') + : fs.readFileSync(manifestPath, 'utf8') + const manifest = JSON.parse(raw) as { + segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }> + } + + // THE LAW: every sealed segment declares exactly as many generations as it + // holds frames, and none of them spans the victim. + for (const s of manifest.segments) { + expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames) + expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false) + } + + await reopened.close() + + // And the pass that used to fail on every run now completes: reopen (which + // re-seeds committedRanges from the packed tier) then compact history. + const third = await openBrain(dir) + await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined() + await third.close() + }) + it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 const dir = tempDir() diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts index 27ab85cb..f16e67b3 100644 --- a/tests/unit/db/generation-segments.test.ts +++ b/tests/unit/db/generation-segments.test.ts @@ -147,4 +147,119 @@ describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) + + // ========================================================================== + // THE DENSITY LAW + // ========================================================================== + // + // A sealed segment declares a CONTIGUOUS range and every reader treats that + // range as containment. Folding a sparse batch therefore makes the segment + // claim generations it does not hold — and because `open()` merges declared + // ranges back into committedRanges, the hole is re-admitted as committed + // history and every later maintenance pass fails asking for a frame that was + // never written. That is the "generation N is inside sealed segment + // seg-....bgs's declared range but has no frame — packed history is damaged" + // narration seen on every run of the affected stores. + + it('fold REFUSES a batch with a hole — a dense range may not be declared over sparse input', async () => { + await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow( + /not contiguous: 2 → 4 skips 1 generation/ + ) + // The refusal loses nothing: no segment was sealed, so the generations + // stay in the live tier and the next pass folds them correctly. + expect(store.segments()).toHaveLength(0) + expect(store.hasGeneration(1)).toBe(false) + }) + + it('a wider gap names how many generations it would have swallowed', async () => { + await expect(store.fold([gen(10), gen(20)])).rejects.toThrow( + /not contiguous: 10 → 20 skips 9 generation\(s\)/ + ) + }) + + it('two contiguous runs folded separately declare honest ranges', async () => { + // What the caller now does instead of folding across the gap. + const a = await store.fold([gen(1), gen(2), gen(3)]) + const b = await store.fold([gen(7), gen(8)]) + expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) + expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 }) + // The gap is honestly outside the packed tier. + for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false) + for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true) + expect(await store.actualRanges()).toEqual([ + [1, 3], + [7, 8] + ]) + }) + + it('actualRanges() is exact and I/O-free for dense segments', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + // Adjacent dense segments each contribute their declared range. + expect(await store.actualRanges()).toEqual([ + [1, 2], + [3, 4] + ]) + }) + + // ---- pre-existing damage: a store sealed by the old writer ---------------- + + /** + * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and + * sidecar for a contiguous run, then rewrite the manifest so the segment + * declares a wider range than the frames it holds. This reproduces on disk + * exactly what the affected stores carry, without needing the old code. + */ + const sealSparseSegment = async (): Promise => { + await store.fold([gen(1), gen(2), gen(3)]) + const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any + // Declare 1..5 while holding frames for 1..3 — generations 4 and 5 become + // holes inside a sealed range. + manifest.segments[0].lastGeneration = 5 + await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest) + } + + it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + + // The frames it really holds still serve, byte-faithfully. + expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002) + expect(await reopened.readRecords(3)).toHaveLength(2) + + // The holes answer "not packed" instead of throwing. This is the fix for + // the wedge: the old reader threw here on EVERY maintenance pass. + expect(await reopened.readDelta(4)).toBeNull() + expect(await reopened.readRecords(5)).toBeNull() + }) + + it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + // Declared 1..5; actually holds 1..3. The store seeds committedRanges from + // THIS, so generations 4 and 5 never become committed history again. + expect(await reopened.actualRanges()).toEqual([[1, 3]]) + }) + + it('a DENSE segment missing a frame is still loud damage', async () => { + // The other side of the branch: when the manifest claims a complete span, + // a missing frame means the manifest and sidecar disagree — real damage, + // and it must not be quietly downgraded to "unpacked". + await store.fold([gen(1), gen(2), gen(3)]) + const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` + const raw = (await storage.readRawBytes(idxPath))! + const { decode, encode } = await import('@msgpack/msgpack') + const idx = decode(raw) as any + // Drop generation 2's entry while the manifest still declares 3 frames. + idx.generations = idx.generations.filter(([g]: [number]) => g !== 2) + await storage.writeRawBytes(idxPath, encode(idx)) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + await expect(reopened.readDelta(2)).rejects.toThrow( + /manifest and the sidecar disagree; packed history is damaged/ + ) + }) }) From 655aa13ea79e23927cde7fd47ab13b505cb042d9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:30:46 -0700 Subject: [PATCH 070/150] =?UTF-8?q?build(release):=20the=20docs-push=20ste?= =?UTF-8?q?p=20retires=20=E2=80=94=20this=20engine=20documents=20itself=20?= =?UTF-8?q?in=20its=20own=20repository?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one-doc-set ruling (2026-08-31) gives soulcraft.com/docs to the paid product alone; the site serves redirects for the slugs this rail used to push. The push script stays in the tree as history; the rail stops calling it. --- scripts/release.sh | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 67ad1053..5d434320 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -248,17 +248,12 @@ else echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" fi -# Step 12: Push public docs to the soulcraft.com docs ingest door -# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when -# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — -# that already happened) when a push errors, so the docs site never -# silently trails npm. -echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" -if node scripts/push-docs.js; then - echo -e "${GREEN}✅ Docs push step done${NC}\n" -else - echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" -fi +# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): +# soulcraft.com/docs carries the paid product's documentation only. This +# engine's documentation home is THIS repository — README and docs/ — and the +# site serves 301s for the slugs this rail used to push. The push script stays +# in the tree for history; the rail no longer calls it. +echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" From c99308710aa5030d80bef794354216851443916c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:07:18 -0700 Subject: [PATCH 071/150] fix(recovery): a torn generation-log tail is a terminal verdict, never a wait MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of one defect, found by a seeded-SIGKILL crash lane. THE FALSE POSITIVE. stampEntityTree() recorded generationStore.generation() — the ALLOCATED counter, a number a write in flight has claimed and may never commit — while the JSDoc beside it already said the source is the committed generation. Every crash inside a write window therefore produced a spurious verdict at the next open: either 'sourceGeneration N is ahead of the log head N-1' (the allocated generation died with the process) or 'rollup invariant nounCount: stamped X, observed Y' (the recovery fold folded facts the stamp's counts predate). Both told the operator to run repairIndex() — a whole-store recount — for a store that was coherent. Measured before this commit: 4 of 11 SIGKILL cycles on a healthy store raised one of the two. The stamp and the open now both read committedGeneration(), which is what every other open-time watermark in the class already reasons about. THE TERMINAL VERDICT. A stamp still ahead of committed truth after the recovery fold witnesses a generation that is not in the log — the stamp's fsync outlived the tail's, and there is nothing to arrive. That is its own verdict state now ('torn'), never folded in with 'incoherent': the two have opposite cures. A writer open demotes it — the unusable stamped surface is re-derived at the committed generation from the live counters, O(1), straight-line, no loop and no await on external progress, narrated with both count sets, the stamp's path and its committedAt. A read-only open cannot re-stamp, so it says so and names the cure instead of guessing, and still serves. Neither branch waits, and neither locks an owner out of a canonical tree the stamp only describes. Pins: the verifier returns the torn verdict with both generations; a fabricated head-behind-source store narrates precisely, demotes inside a bounded open, serves its rows, and is quiet at the next open (the demotion converges); a read-only open narrates the same verdict and leaves the bytes untouched. (cherry picked from commit 298cb6dacaac9ef80db65a723d65cbd53c15d23e) --- src/brainy.ts | 121 +++++++++++++++++++- src/db/familyStamp.ts | 32 ++++-- tests/integration/entity-tree-stamp.test.ts | 104 ++++++++++++++++- 3 files changed, 241 insertions(+), 16 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 70d46973..da04577e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -12497,6 +12497,18 @@ export class Brainy implements BrainyInterface { * healed by `repairIndex()`, whose unconditional recount rebuilds the * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write * fault warns loudly but never fails the flush that carried real data. + * + * THE SOURCE IS `committedGeneration()`, NEVER `generation()`. The latter is + * the ALLOCATED counter — a number a write in flight has claimed and may + * never commit. Stamping it made the stamp's generation label a claim about + * counts it was not taken at, and every crash inside a write window then + * produced a spurious verdict at the next open: either `sourceGeneration N + * is ahead of the log head N-1` (the allocated generation died with the + * process) or `rollup invariant 'nounCount': stamped X, observed Y` (the + * recovery fold folded facts the stamp's counts predate). MEASURED on the + * crash-consistency lane before this line changed: 4 of 11 SIGKILL cycles on + * a coherent store raised one of those two verdicts, each of them naming + * `repairIndex()` — a whole-store recount — as the cure for nothing. */ private async stampEntityTree(): Promise { if (this.isReadOnly) return @@ -12507,7 +12519,7 @@ export class Brainy implements BrainyInterface { ]) await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { family: 'entity-tree', - sourceGeneration: this.generationStore.generation(), + sourceGeneration: this.generationStore.committedGeneration(), members: { mode: 'rollup', invariants: { nounCount, verbCount } } }) } catch (error) { @@ -12520,16 +12532,24 @@ export class Brainy implements BrainyInterface { /** * @description Open-time coherence check for the entity tree's family stamp: - * compare `sourceGeneration` against the log head and the stamped rollup - * invariants against the live counters. Verdicts: + * compare `sourceGeneration` against the store's COMMITTED generation and + * the stamped rollup invariants against the live counters. Verdicts: * - `coherent` / `absent` (legacy store; first flush stamps) → silent. * - `behind` → benign for the tree (it is written BY the commit; only the * stamp is stale — a crash landed between commit and flush). Refreshed at * the next flush. + * - `torn` → a TORN GENERATION-LOG TAIL, handled by + * {@link demoteTornEntityTreeStamp}: terminal, never a wait. * - `incoherent` → LOUD: the tree or its counters diverged from what was * stamped — `repairIndex()` recounts from canonical and re-stamps. * Never blocks open; a fault reading the stamp is surfaced as unverifiable, * never conflated with absence. + * + * THE COMPARISON IS AGAINST `committedGeneration()`, matching what + * {@link stampEntityTree} writes and what every other open-time watermark in + * this class already reasons about (the fact-scan capability, the metadata / + * graph / HNSW watermark verdicts). Comparing against the allocated counter + * was the one place that disagreed, and disagreeing was the whole defect. */ private async verifyEntityTreeStamp(): Promise { let stamp: FamilyStamp | null @@ -12546,11 +12566,16 @@ export class Brainy implements BrainyInterface { this.storage.getNounCount(), this.storage.getVerbCount() ]) - const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { + const verdict = verifyFamilyStamp(stamp, this.generationStore.committedGeneration(), { nounCount, verbCount }) - if (verdict.state === 'incoherent') { + if (verdict.state === 'torn') { + await this.demoteTornEntityTreeStamp(stamp as FamilyStamp, verdict.stampSource, verdict.head, { + nounCount, + verbCount + }) + } else if (verdict.state === 'incoherent') { prodLog.warn( `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + `The canonical tree or its counters diverged from the stamped state — run ` + @@ -12564,6 +12589,92 @@ export class Brainy implements BrainyInterface { } } + /** + * @description THE TERMINAL VERDICT for a torn generation-log tail. + * + * A stamp whose `sourceGeneration` sits ABOVE the store's committed + * watermark witnesses a generation that is not in the log: the stamp's fsync + * outlived the tail's. By the time this runs, log-authority recovery has + * already folded every intact fact above the manifest and advanced the + * watermark to cover them — so if the stamp is STILL ahead, the generation + * it names is not merely late, it is GONE. There is nothing to wait for. + * + * That is the whole point of this method. A field report of this class + * (single-process store, abrupt termination mid-fold) described a reopen + * that narrated the tear and then held 100% CPU with zero log growth for + * eight minutes before an operator wiped the directory. A recovery that + * cannot say what it is waiting for has no business spinning; the honest + * answer here is a verdict, taken now, at O(1) cost. + * + * WHAT THE VERDICT DOES — the stamped surface is UNUSABLE, so it is + * discarded rather than believed: the stamped counts describe a generation + * that never became durable, and comparing them against live counters can + * only produce noise. The tree itself is not in question (it IS canonical — + * every commit writes it, and the fold re-applied every after-image the log + * still holds), so the demotion is a re-derivation of this family's verified + * surface at the generation the store can actually show: + * + * - WRITER open → re-stamp at `committedGeneration()` from the live + * counters — exactly what the next flush would write, taken now so the + * tear cannot re-narrate on every subsequent open. Both count sets are + * logged so an operator can see whether anything really moved. + * - READER open → a reader cannot re-stamp. Narrate the same terminal + * verdict with the named cure and carry on serving; a read-only inspector + * is never locked out of a store, and never left waiting either. + * + * BOUNDEDNESS: straight-line code. No loop, no retry, no await on any + * external progress signal — the two counter reads and one stamp write are + * the entire cost, and none of them scales with the store. + */ + private async demoteTornEntityTreeStamp( + stamp: FamilyStamp, + stampSource: number, + head: number, + observed: { nounCount: number; verbCount: number } + ): Promise { + const stamped = stamp.members.mode === 'rollup' ? stamp.members.invariants : {} + const detail = + `[Brainy] TORN GENERATION-LOG TAIL at open: ${ENTITY_TREE_STAMP_PATH} witnesses source ` + + `generation ${stampSource} (stamped ${stamp.committedAt}), but the store's committed ` + + `generation is ${head} after crash recovery — the stamp's fsync outlived the log tail's, ` + + `and generation ${stampSource} is not in the log to arrive. Stamped rollups ` + + `${JSON.stringify(stamped)}; observed ${JSON.stringify(observed)}.` + + if (this.isReadOnly) { + prodLog.warn( + `${detail} This open is READ-ONLY, so the stamp cannot be re-derived: the entity-tree ` + + `family stays UNVERIFIED for this session (reads are unaffected — the canonical tree ` + + `is the truth this stamp only describes). Cure: open the store with a writer, or run ` + + `brain.repairIndex() there, to recount from canonical and re-stamp.` + ) + return + } + + const startedAt = Date.now() + try { + await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { + family: 'entity-tree', + sourceGeneration: head, + members: { + mode: 'rollup', + invariants: { nounCount: observed.nounCount, verbCount: observed.verbCount } + } + }) + prodLog.warn( + `${detail} DEMOTED: the unusable stamp was re-derived at committed generation ${head} ` + + `from the live counters in ${Date.now() - startedAt}ms — terminal, not a wait. If the ` + + `observed counts above look wrong for your data, run brain.repairIndex() to recount ` + + `from canonical.` + ) + } catch (error) { + prodLog.warn( + `${detail} The demotion's re-stamp FAILED (${(error as Error).message}) — the tear will ` + + `narrate again at the next open, which is the honest outcome; the store still serves ` + + `from canonical. Cure: run brain.repairIndex() to recount from canonical and re-stamp.` + ) + } + } + /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. diff --git a/src/db/familyStamp.ts b/src/db/familyStamp.ts index 98342884..2f01e935 100644 --- a/src/db/familyStamp.ts +++ b/src/db/familyStamp.ts @@ -12,9 +12,11 @@ * the verified surface is a small set of rollup invariants (entity/ * relationship counts) plus `sourceGeneration`. * - * `sourceGeneration` is the generation of the source-of-truth log this - * projection reflects — open-time coherence becomes a COMPARISON (stamp vs - * log head), not a walk: + * `sourceGeneration` is the COMMITTED generation of the source-of-truth log + * this projection reflects — never the allocated counter, which names a + * generation that may never commit (see {@link StampVerdict.torn}) — so + * open-time coherence becomes a COMPARISON (stamp vs committed head), not a + * walk: * * - equal + invariants hold → coherent, serve. * - behind → the projection missed the tail (crash between commit and stamp); @@ -24,6 +26,9 @@ * - invariants FAIL at equal generation → genuine incoherence: loud, and the * repair ritual (`repairIndex()`, whose recount rebuilds the rollups from a * canonical walk) heals it. + * - AHEAD → a torn generation-log tail: the stamp's fsync outlived the log + * tail's. TERMINAL, never a wait — the generation the stamp names does not + * exist to arrive. * * Stamps are JSON on purpose — every incident gets debugged by reading a * stamp in a terminal. @@ -70,6 +75,12 @@ export type StampVerdict = | { state: 'coherent' } | { state: 'absent' } // legacy store — first stamp writes at the next flush | { state: 'behind'; stampSource: number; head: number } + /** + * TORN GENERATION-LOG TAIL: the stamp witnesses a source generation the + * store's committed watermark can no longer show. TERMINAL — there is no + * generation to wait for, so the open demotes (or refuses) and never spins. + */ + | { state: 'torn'; stampSource: number; head: number } | { state: 'incoherent'; failures: string[] } | { state: 'unverifiable'; reason: string } // a FAULT reading the stamp — never conflated with absence @@ -118,12 +129,15 @@ export function verifyFamilyStamp( ): StampVerdict { if (stamp === null) return { state: 'absent' } if (stamp.sourceGeneration > head) { - // A stamp AHEAD of the log claims state that never committed — the - // projection was stamped against truth that a crash rolled back. - return { - state: 'incoherent', - failures: [`sourceGeneration ${stamp.sourceGeneration} is ahead of the log head ${head}`] - } + // A stamp AHEAD of committed truth witnesses a generation the store can no + // longer show: the stamp's fsync survived a crash that the log tail did + // not. This is the TORN GENERATION-LOG TAIL — its own class, never folded + // in with `incoherent` (a count that drifted at a generation both sides + // agree on), because the two have opposite cures: incoherence is recounted, + // a tear is DEMOTED. It is also terminal by construction — there is no + // generation the open can wait for, because the one the stamp names is + // gone. + return { state: 'torn', stampSource: stamp.sourceGeneration, head } } if (stamp.sourceGeneration < head) { return { state: 'behind', stampSource: stamp.sourceGeneration, head } diff --git a/tests/integration/entity-tree-stamp.test.ts b/tests/integration/entity-tree-stamp.test.ts index deefc5e6..23cc0a15 100644 --- a/tests/integration/entity-tree-stamp.test.ts +++ b/tests/integration/entity-tree-stamp.test.ts @@ -57,7 +57,11 @@ describe('entity-tree family stamp', () => { const invariants = (stamp.members as any).invariants expect(invariants.nounCount).toBe(await brain.storage.getNounCount()) expect(invariants.verbCount).toBe(await brain.storage.getVerbCount()) - expect(stamp.sourceGeneration).toBe(brain.generation()) + // THE SOURCE IS COMMITTED TRUTH, never the allocated counter. Stamping the + // counter labelled the stamp with a generation a write in flight had merely + // claimed, so every crash inside a write window produced a spurious verdict + // at the next open (see the torn-tail pins below). + expect(stamp.sourceGeneration).toBe(brain.generationStore.committedGeneration()) expect(stamp.generation).toBeGreaterThanOrEqual(1) }) @@ -112,6 +116,96 @@ describe('entity-tree family stamp', () => { expect(stillIncoherent).toEqual([]) }) + /** + * Rewrite the on-disk stamp so its `sourceGeneration` sits ABOVE the store's + * committed watermark — the durable shape a torn generation-log tail leaves + * behind (the stamp's fsync outlived the tail's). Fabricated rather than + * crash-produced so the pin is deterministic; the seeded-SIGKILL lane + * (`scripts/crash-consistency.mjs` in the engine repo) produces the same + * shape from a real abrupt termination. + */ + const fabricateTear = (ahead: number): FamilyStamp => { + const file = path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`) + const zlib = require('node:zlib') + const raw = JSON.parse(zlib.gunzipSync(fs.readFileSync(file)).toString('utf-8')) as FamilyStamp + const torn: FamilyStamp = { ...raw, sourceGeneration: raw.sourceGeneration + ahead } + fs.writeFileSync(file, zlib.gzipSync(JSON.stringify(torn))) + return torn + } + + it('a torn generation-log tail is a TERMINAL VERDICT at open: narrated, demoted, never a wait', async () => { + for (let i = 0; i < 3; i++) + await brain.add({ data: `torn${i}`, type: 'document', metadata: { i } }) + await brain.close() + const torn = fabricateTear(5) + + const warn = vi.spyOn(prodLog, 'warn') + const startedAt = Date.now() + brain = await open() + const openMs = Date.now() - startedAt + + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + // Narrated PRECISELY: both generations, the file, and the named cure. + expect(said).toContain(`source generation ${torn.sourceGeneration}`) + expect(said).toContain(`committed generation ${brain.generationStore.committedGeneration()}`) + expect(said).toContain(ENTITY_TREE_STAMP_PATH) + expect(said).toContain('DEMOTED') + expect(said).toMatch(/repairIndex\(\)/) + // Terminal, not a wait: the demotion is O(1) straight-line work, so a tear + // cannot turn an open into the 8-minute spin this class was reported as. + expect(openMs).toBeLessThan(30_000) + + // The store SERVES — a tear in a stamp never locks an owner out of the + // canonical tree the stamp merely describes. + expect((await brain.find({ type: 'document', limit: 100 })).length).toBe(3) + + // The demotion CONVERGED: the stamp now names committed truth, and the + // next open is quiet. A verdict that re-narrates every open is a wait + // wearing a different hat. + const restamped = (await readFamilyStamp(brain.storage, ENTITY_TREE_STAMP_PATH)) as FamilyStamp + expect(restamped.sourceGeneration).toBe(brain.generationStore.committedGeneration()) + await brain.close() + const warn2 = vi.spyOn(prodLog, 'warn') + brain = await open() + expect(warn2.mock.calls.filter((c) => String(c[0]).includes('TORN'))).toEqual([]) + }) + + it('a READ-ONLY open on a torn tail refuses to guess: terminal verdict + named cure, no re-stamp', async () => { + await brain.add({ data: 'ro', type: 'document', metadata: {} }) + await brain.close() + const torn = fabricateTear(3) + + const warn = vi.spyOn(prodLog, 'warn') + const reader: any = await Brainy.openReadOnly({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + const tearLines = warn.mock.calls.filter((c) => String(c[0]).includes('TORN GENERATION-LOG TAIL')) + expect(tearLines.length).toBe(1) + const said = String(tearLines[0][0]) + expect(said).toContain('READ-ONLY') + expect(said).toContain('UNVERIFIED') + expect(said).toMatch(/repairIndex\(\)/) + await reader.close() + + // A reader never rewrites the store: read the bytes back off disk (not + // through a writer open, which would demote them) — the torn stamp is + // exactly as it was found. + const onDisk = JSON.parse( + require('node:zlib') + .gunzipSync(fs.readFileSync(path.join(dir, `${ENTITY_TREE_STAMP_PATH}.gz`))) + .toString('utf-8') + ) as FamilyStamp + expect(onDisk.sourceGeneration).toBe(torn.sourceGeneration) + expect(onDisk.generation).toBe(torn.generation) + + brain = await open() + }) + it('the one verifier handles both member modes', () => { const rollup: FamilyStamp = { family: 'x', @@ -127,7 +221,13 @@ describe('entity-tree family stamp', () => { stampSource: 5, head: 9 }) - expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 }).state).toBe('incoherent') // ahead of head + // AHEAD is its own class — a torn generation-log tail, never folded in + // with `incoherent`: the two have opposite cures (recount vs demote). + expect(verifyFamilyStamp(rollup, 3, { nounCount: 10 })).toEqual({ + state: 'torn', + stampSource: 5, + head: 3 + }) expect(verifyFamilyStamp(null, 5, {})).toEqual({ state: 'absent' }) const enumerated: FamilyStamp = { From a963a744ccf668edc440c76d7f79fd0b216522c1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:13:42 -0700 Subject: [PATCH 072/150] fix(generations): a sealed segment may only declare the generations it holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis of the "packed history is damaged" narration that fires on every run of the affected stores. It is a WRITER defect, and the reader's refusal was the symptom rather than the cause. A sealed segment declares one contiguous range [firstGeneration, lastGeneration], and every reader treats that range as containment: coveringSegment is an interval test, hasGeneration returns true for anything inside it, and open() seeds committedRanges from it. repackHistory handed fold() a SPARSE batch. Three filters punch holes in its candidate list mid-run — a generation absent from committedRanges never appears, one still in the pending buffer is skipped, one whose tx.json will not read is skipped — and fold() then computed the range from the first and last survivor, claiming every generation in between. The next open merged that mis-declared range back into committedRanges, re-admitting the hole as committed history, so the following auto-compaction pass asked the packed tier for a frame that was never written and failed. Re-merged at every open, which is why it repeated on every run. Confirmed against a forensic fixture: generation directories 1..2503 present except exactly one, 1416; and its fact-log segment already showed the tell — seg-...1410.bfl declaring 1410..1940 (531 generations) while recording 530 facts. Three changes: - repackHistory folds each contiguous RUN as its own segment (`contiguousRuns`), so ranges describe exactly what the segments contain. - fold() REFUSES a non-contiguous batch, naming the gap and its width. The density law is now mechanical, so no future caller can reintroduce it. A refusal loses nothing: the generations stay live and readable. - Stores already carrying the damage heal instead of wedging. A segment whose declared span exceeds its frame count is SPARSE; `actualRanges()` reads the real generation list from its sidecar so open() never re-admits the holes, and readFrame reports such a hole as unpacked with a narration naming the segment, rather than throwing. A DENSE segment missing a frame is still loud damage — that one means the manifest and sidecar disagree. Pins: nine unit cases (refusal and its message, honest ranges for separately folded runs, a reconstructed pre-fix sparse segment serving its real frames while reporting holes as unpacked, holes excluded from actualRanges, and the dense-segment damage path still throwing) plus an end-to-end case that deletes a generation directory and drives the real sequence — ordinary close()-time repacking folds over the hole, then reopen and compact must both complete. Verified red without the fix: the segment declared an 11-generation span while holding 10 frames. (cherry picked from commit 9a888c37e9ebec5573cd7ebd0764396f3a424de3) --- src/db/generationSegments.ts | 119 +++++++++++++++++++- src/db/generationStore.ts | 66 +++++++++-- tests/integration/history-repacking.test.ts | 102 +++++++++++++++++ tests/unit/db/generation-segments.test.ts | 115 +++++++++++++++++++ 4 files changed, 389 insertions(+), 13 deletions(-) diff --git a/src/db/generationSegments.ts b/src/db/generationSegments.ts index 0c14b60c..91451281 100644 --- a/src/db/generationSegments.ts +++ b/src/db/generationSegments.ts @@ -147,6 +147,60 @@ export class GenerationSegmentStore { return this.coveringSegment(gen) !== null } + /** + * @description True when `meta` declares more generations than it holds + * frames — a segment sealed by a writer that folded across a hole. The + * manifest records `frames` at fold time, so this is an O(1) comparison + * against the declared span and needs no I/O. + */ + private isSparse(meta: SegmentMeta): boolean { + return meta.lastGeneration - meta.firstGeneration + 1 !== meta.frames + } + + /** + * @description The generations this tier ACTUALLY holds, as coalesced + * ascending intervals — not what the segments declare. + * + * Dense segments (every one a current writer produces) contribute their + * declared range with no I/O. A SPARSE segment — one sealed before the + * density law was enforced, whose declared range spans generations it has + * no frame for — has its real generation list read from its sidecar and + * contributed instead, with the discrepancy narrated once. + * + * This is what keeps a store that already carries the damage from wedging. + * `open()` seeds `committedRanges` from these intervals, so a hole is never + * re-admitted as a committed generation, and the auto-compaction pass that + * used to fail on every run with "packed history is damaged" simply never + * asks for the missing frame. + * + * @returns Ascending, non-overlapping `[first, last]` intervals. + */ + async actualRanges(): Promise> { + const out: Array<[number, number]> = [] + for (const meta of this.manifest.segments) { + if (!this.isSparse(meta)) { + out.push([meta.firstGeneration, meta.lastGeneration]) + continue + } + const missing = meta.lastGeneration - meta.firstGeneration + 1 - meta.frames + prodLog.warn( + `[GenerationSegments] sealed segment ${meta.file} declares generations ` + + `${meta.firstGeneration}..${meta.lastGeneration} but holds only ${meta.frames} ` + + `frame(s) — ${missing} generation(s) in that span were never folded into it. ` + + `Serving the frames it actually holds; the declared span is not treated as ` + + `committed history. (Written by a pre-density-law writer that folded across a ` + + `gap; the segment itself is intact and no record is lost.)` + ) + const idx = await this.sidecarFor(meta) + for (const [gen] of idx.generations) { + const last = out[out.length - 1] + if (last !== undefined && gen === last[1] + 1) last[1] = gen + else out.push([gen, gen]) + } + } + return out + } + /** * Fold consecutive generations into ONE new sealed segment + sidecar and * append it to the manifest atomically. Caller guarantees: `gens` is @@ -164,6 +218,38 @@ export class GenerationSegmentStore { throw new Error('[GenerationSegments] fold() input must be strictly ascending') } } + // THE DENSITY LAW, MADE MECHANICAL. + // + // A sealed segment declares a CONTIGUOUS range [firstGeneration, + // lastGeneration] and every reader treats that range as containment: + // `coveringSegment` is an interval test, `hasGeneration` returns true for + // anything inside it, and `open()` seeds committedRanges from it. So a + // segment folded from a SPARSE input silently claims generations it does + // not hold, and the first read of one of those holes throws + // "inside sealed segment ... but has no frame — packed history is damaged". + // + // That is exactly how the damage was produced. `repackHistory` skipped + // generations mid-batch — ones absent from committedRanges, ones still in + // the pending buffer, ones whose tx.json would not read — and handed the + // survivors here, where the range was computed from the first and last of + // them. Worse, the mis-declared range was then merged back into + // committedRanges at the next open, which is what turned a quiet hole into + // a repeating auto-compaction failure on every subsequent run. + // + // Callers now split at discontinuities; this refusal is what keeps any + // future caller from reintroducing the class. A refusal here loses + // nothing — the generations stay in the live tier, readable, and the next + // pass folds them correctly. + for (let i = 1; i < gens.length; i++) { + if (gens[i].generation !== gens[i - 1].generation + 1) { + throw new Error( + `[GenerationSegments] fold() input is not contiguous: ${gens[i - 1].generation} → ` + + `${gens[i].generation} skips ${gens[i].generation - gens[i - 1].generation - 1} ` + + `generation(s). A sealed segment declares a dense range, so folding a sparse ` + + `batch would claim generations it does not hold. Split the batch at the gap.` + ) + } + } const last = this.manifest.segments[this.manifest.segments.length - 1] if (last && gens[0].generation <= last.lastGeneration) { throw new Error( @@ -364,12 +450,37 @@ export class GenerationSegmentStore { return this.decodeFrame(payload) } } - // In the covering range but not present: the packed tier is dense by - // construction (fold packs every generation it is handed, including - // record-less ones) — absence inside a sealed range is damage. + // Inside the covering range but with no frame. Two very different causes, + // and conflating them is what made this class wedge every maintenance pass + // on the affected stores. + // + // (1) A SPARSE SEGMENT — the manifest's own `frames` count is smaller than + // the span it declares. That segment was sealed by a writer that + // folded across a hole (the class this file's density law now bars). + // The segment is INTACT and nothing is lost; it simply never held this + // generation. Answering "not packed" is the honest answer, and it lets + // the caller's two-tier read decide what a genuinely absent generation + // means, instead of every compaction pass dying on a repeating throw. + // `actualRanges()` keeps such holes out of committedRanges at open, so + // in a healed store nobody asks this question in the first place. + // + // (2) A DENSE SEGMENT missing a frame it says it has — the manifest and + // the sidecar disagree about a segment that claims to be complete. + // That IS damage, and it stays loud. + if (this.isSparse(meta)) { + prodLog.warn( + `[GenerationSegments] generation ${gen} falls inside sealed segment ${meta.file}'s ` + + `declared range ${meta.firstGeneration}..${meta.lastGeneration}, but that segment ` + + `holds ${meta.frames} frame(s) for a ${meta.lastGeneration - meta.firstGeneration + 1}` + + `-generation span — it was sealed across a gap and never held this generation. ` + + `Reporting it as unpacked rather than as damage; no record is lost.` + ) + return null + } throw new Error( `[GenerationSegments] generation ${gen} is inside sealed segment ${meta.file}'s declared ` + - `range but has no frame — packed history is damaged` + `range but has no frame, and that segment declares a complete ${meta.frames}-frame ` + + `span — the manifest and the sidecar disagree; packed history is damaged` ) } diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index fd052c31..da21dc61 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -96,6 +96,35 @@ export const FOLD_CHECKPOINT_PATH = '_system/fold-checkpoint.json' /** Storage-root-relative prefix of the per-generation record directories. */ export const GENERATIONS_PREFIX = '_generations' +/** + * @description Split an ascending list of fold candidates into maximal + * CONTIGUOUS runs — `[7,8,9,12,13]` becomes `[[7,8,9],[12,13]]`. + * + * A sealed segment declares one dense range `[firstGeneration, + * lastGeneration]`, and every reader treats that range as containment. So a + * batch with a hole in it must never become one segment: it would claim a + * generation it does not hold, and the first read of that hole reports the + * packed history as damaged. One run, one segment — the ranges then describe + * exactly what the segments contain. + * + * @param gens - Fold candidates, strictly ascending by generation. + * @returns One array per contiguous run, in ascending order. Empty in, empty out. + */ +export function contiguousRuns(gens: FoldGeneration[]): FoldGeneration[][] { + const runs: FoldGeneration[][] = [] + let run: FoldGeneration[] = [] + for (const g of gens) { + const prev = run[run.length - 1] + if (prev !== undefined && g.generation !== prev.generation + 1) { + runs.push(run) + run = [] + } + run.push(g) + } + if (run.length > 0) runs.push(run) + return runs +} + /** * @description Phases of the {@link GenerationStore.commitTransaction} commit * protocol at which a test-only fault injector can simulate a process crash. @@ -784,9 +813,15 @@ export class GenerationStore { if (storageSupportsFactLog(this.storage)) { this.segments = new GenerationSegmentStore(this.storage) await this.segments.open() - const packedRanges = this.segments - .segments() - .map((s): [number, number] => [s.firstGeneration, Math.min(s.lastGeneration, this.committed)]) + // ACTUAL ranges, not declared ones. A segment sealed by a pre-density-law + // writer can declare a span wider than the frames it holds; seeding + // committedRanges from the declared span re-admits those holes as + // committed generations, and every later maintenance pass then asks for a + // frame that was never written. `actualRanges()` reads the real + // generation list from the sidecar for exactly those segments (and does + // no I/O for the dense ones, which is all of them on a healthy store). + const packedRanges = (await this.segments.actualRanges()) + .map((r): [number, number] => [r[0], Math.min(r[1], this.committed)]) .filter(([lo, hi]) => lo <= hi) if (packedRanges.length > 0) { // Merge packed (older) + live (newer) interval sets — both ascending; @@ -3121,13 +3156,26 @@ export class GenerationStore { foldInput.push({ generation: gen, timestamp: delta.timestamp, delta, records }) } if (foldInput.length === 0) continue - await segments.fold(foldInput) - segmentsCreated++ - // Segment + manifest durable → the live copies retire. - for (const g of foldInput) { - await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + // SPLIT AT DISCONTINUITIES. `eligible` is NOT contiguous — three + // filters above punch holes in it: a generation missing from + // committedRanges never appears, one still in the pending buffer is + // skipped, and one whose tx.json will not read is skipped. A sealed + // segment declares a DENSE range, so folding across such a hole makes + // the segment claim a generation it does not hold; the next open + // merges that mis-declared range into committedRanges, and every + // subsequent auto-compaction pass then asks for the missing frame and + // fails with "packed history is damaged". Fold each contiguous RUN as + // its own segment instead — same bytes, honest ranges. + for (const run of contiguousRuns(foldInput)) { + if (deadline !== undefined && Date.now() >= deadline) break + await segments.fold(run) + segmentsCreated++ + // Segment + manifest durable → the live copies retire. + for (const g of run) { + await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${g.generation}`) + } + folded += run.length } - folded += foldInput.length } if (folded > 0) { prodLog.info( diff --git a/tests/integration/history-repacking.test.ts b/tests/integration/history-repacking.test.ts index 2bcee038..bb07268d 100644 --- a/tests/integration/history-repacking.test.ts +++ b/tests/integration/history-repacking.test.ts @@ -16,6 +16,7 @@ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as path from 'node:path' import * as os from 'node:os' +import * as zlib from 'node:zlib' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { GenerationStore } from '../../src/db/generationStore.js' @@ -57,6 +58,107 @@ describe('history repacking — the two-tier lifecycle', () => { } }) + /** + * THE HOLE, END TO END — the shape a real store carries. + * + * A forensic fixture was measured with generation directories 1..2503 + * present except for exactly one: 1416. Its fact-log segment already showed + * the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration + * 1940 (531 generations) while recording only 530 facts. + * + * Before the fix, repacking such a store folded ACROSS that hole: the batch + * skipped 1416 (no readable delta) and the sealed segment declared a range + * spanning it anyway. The next open merged that declared range back into + * committedRanges, re-admitting 1416 as committed history, and every + * subsequent auto-compaction pass then asked the packed tier for a frame + * that was never written — producing, on EVERY run, the non-fatal narration + * + * Auto-compaction of generational history failed (non-fatal): generation + * N is inside sealed segment seg-....bgs's declared range but has no frame + * — packed history is damaged + * + * This pin removes a generation directory to make the same hole, then + * requires repack + reopen + compaction to complete cleanly. + */ + it('a missing generation directory does not poison the packed tier', async () => { + const dir = tempDir() + // `retention: 'all'` throughout: close() otherwise auto-compacts the + // history away, and this pin needs the cold generations still on disk so + // there is something to punch a hole in. The live window stays at its + // production default for the build phase, so nothing folds yet. + const archival = async (): Promise => { + const b = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + embeddingFunction: stub, + retention: 'all' + }) + await b.init() + return b + } + const brain = await archival() + + const id = await brain.add({ + data: 'holed-entity', + type: NounType.Document, + metadata: { v: 0 } + }) + // One flush per update: single-op writes coalesce inside a flush window, + // so a history deep enough to have a middle needs the windows separated. + for (let v = 1; v <= 12; v++) { + await brain.update({ id, metadata: { v } }) + await brain.flush() + } + await brain.close() + + // Punch the hole: delete ONE generation directory in the middle of the + // cold range, exactly as the real store presents it. + const genRoot = path.join(dir, '_generations') + const numeric = fs + .readdirSync(genRoot, { withFileTypes: true }) + .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)) + .map((e) => Number(e.name)) + .sort((a, b) => a - b) + expect(numeric.length).toBeGreaterThan(6) + const victim = numeric[Math.floor(numeric.length / 2)] + fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true }) + + // Now shrink the live window and reopen. close() repacks automatically + // (brainy.ts phase 0b), so this is the production sequence exactly: a + // store with a hole in its history gets folded by ordinary housekeeping, + // with nobody asking for it. + ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 + const reopened = await archival() + const result = await reopened.repackHistory() + expect(result.foldedGenerations).toBeGreaterThan(0) + + const segDir = path.join(dir, SEGMENTS_PREFIX) + const manifestPath = ['manifest.json', 'manifest.json.gz'] + .map((f) => path.join(segDir, f)) + .find((p) => fs.existsSync(p))! + const raw = manifestPath.endsWith('.gz') + ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8') + : fs.readFileSync(manifestPath, 'utf8') + const manifest = JSON.parse(raw) as { + segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }> + } + + // THE LAW: every sealed segment declares exactly as many generations as it + // holds frames, and none of them spans the victim. + for (const s of manifest.segments) { + expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames) + expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false) + } + + await reopened.close() + + // And the pass that used to fail on every run now completes: reopen (which + // re-seeds committedRanges from the packed tier) then compact history. + const third = await openBrain(dir) + await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined() + await third.close() + }) + it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => { ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3 const dir = tempDir() diff --git a/tests/unit/db/generation-segments.test.ts b/tests/unit/db/generation-segments.test.ts index 27ab85cb..f16e67b3 100644 --- a/tests/unit/db/generation-segments.test.ts +++ b/tests/unit/db/generation-segments.test.ts @@ -147,4 +147,119 @@ describe('db/GenerationSegmentStore — the D1+D3 packed tier', () => { await expect(store.fold([gen(4), gen(4)])).rejects.toThrow(/strictly ascending/) await expect(store.fold([])).rejects.toThrow(/at least one generation/) }) + + // ========================================================================== + // THE DENSITY LAW + // ========================================================================== + // + // A sealed segment declares a CONTIGUOUS range and every reader treats that + // range as containment. Folding a sparse batch therefore makes the segment + // claim generations it does not hold — and because `open()` merges declared + // ranges back into committedRanges, the hole is re-admitted as committed + // history and every later maintenance pass fails asking for a frame that was + // never written. That is the "generation N is inside sealed segment + // seg-....bgs's declared range but has no frame — packed history is damaged" + // narration seen on every run of the affected stores. + + it('fold REFUSES a batch with a hole — a dense range may not be declared over sparse input', async () => { + await expect(store.fold([gen(1), gen(2), gen(4)])).rejects.toThrow( + /not contiguous: 2 → 4 skips 1 generation/ + ) + // The refusal loses nothing: no segment was sealed, so the generations + // stay in the live tier and the next pass folds them correctly. + expect(store.segments()).toHaveLength(0) + expect(store.hasGeneration(1)).toBe(false) + }) + + it('a wider gap names how many generations it would have swallowed', async () => { + await expect(store.fold([gen(10), gen(20)])).rejects.toThrow( + /not contiguous: 10 → 20 skips 9 generation\(s\)/ + ) + }) + + it('two contiguous runs folded separately declare honest ranges', async () => { + // What the caller now does instead of folding across the gap. + const a = await store.fold([gen(1), gen(2), gen(3)]) + const b = await store.fold([gen(7), gen(8)]) + expect(a).toMatchObject({ firstGeneration: 1, lastGeneration: 3, frames: 3 }) + expect(b).toMatchObject({ firstGeneration: 7, lastGeneration: 8, frames: 2 }) + // The gap is honestly outside the packed tier. + for (const g of [4, 5, 6]) expect(store.hasGeneration(g)).toBe(false) + for (const g of [1, 2, 3, 7, 8]) expect(store.hasGeneration(g)).toBe(true) + expect(await store.actualRanges()).toEqual([ + [1, 3], + [7, 8] + ]) + }) + + it('actualRanges() is exact and I/O-free for dense segments', async () => { + await store.fold([gen(1), gen(2)]) + await store.fold([gen(3), gen(4)]) + // Adjacent dense segments each contribute their declared range. + expect(await store.actualRanges()).toEqual([ + [1, 2], + [3, 4] + ]) + }) + + // ---- pre-existing damage: a store sealed by the old writer ---------------- + + /** + * Seal a SPARSE segment the way the pre-fix writer did: write the bytes and + * sidecar for a contiguous run, then rewrite the manifest so the segment + * declares a wider range than the frames it holds. This reproduces on disk + * exactly what the affected stores carry, without needing the old code. + */ + const sealSparseSegment = async (): Promise => { + await store.fold([gen(1), gen(2), gen(3)]) + const manifest = (await storage.readRawObject(`${SEGMENTS_PREFIX}/manifest.json`)) as any + // Declare 1..5 while holding frames for 1..3 — generations 4 and 5 become + // holes inside a sealed range. + manifest.segments[0].lastGeneration = 5 + await storage.writeRawObject(`${SEGMENTS_PREFIX}/manifest.json`, manifest) + } + + it('a pre-existing sparse segment reports its holes as UNPACKED, not as damage', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + + // The frames it really holds still serve, byte-faithfully. + expect((await reopened.readDelta(2))?.timestamp).toBe(1_700_000_000_002) + expect(await reopened.readRecords(3)).toHaveLength(2) + + // The holes answer "not packed" instead of throwing. This is the fix for + // the wedge: the old reader threw here on EVERY maintenance pass. + expect(await reopened.readDelta(4)).toBeNull() + expect(await reopened.readRecords(5)).toBeNull() + }) + + it('actualRanges() excludes the holes so they are never re-admitted as committed', async () => { + await sealSparseSegment() + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + // Declared 1..5; actually holds 1..3. The store seeds committedRanges from + // THIS, so generations 4 and 5 never become committed history again. + expect(await reopened.actualRanges()).toEqual([[1, 3]]) + }) + + it('a DENSE segment missing a frame is still loud damage', async () => { + // The other side of the branch: when the manifest claims a complete span, + // a missing frame means the manifest and sidecar disagree — real damage, + // and it must not be quietly downgraded to "unpacked". + await store.fold([gen(1), gen(2), gen(3)]) + const idxPath = `${SEGMENTS_PREFIX}/seg-${String(1).padStart(20, '0')}.idx` + const raw = (await storage.readRawBytes(idxPath))! + const { decode, encode } = await import('@msgpack/msgpack') + const idx = decode(raw) as any + // Drop generation 2's entry while the manifest still declares 3 frames. + idx.generations = idx.generations.filter(([g]: [number]) => g !== 2) + await storage.writeRawBytes(idxPath, encode(idx)) + + const reopened = new GenerationSegmentStore(storage as any) + await reopened.open() + await expect(reopened.readDelta(2)).rejects.toThrow( + /manifest and the sidecar disagree; packed history is damaged/ + ) + }) }) From d6bcb14f698de40507a6ce16daf56ab235f71924 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 09:30:46 -0700 Subject: [PATCH 073/150] =?UTF-8?q?build(release):=20the=20docs-push=20ste?= =?UTF-8?q?p=20retires=20=E2=80=94=20this=20engine=20documents=20itself=20?= =?UTF-8?q?in=20its=20own=20repository?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one-doc-set ruling (2026-08-31) gives soulcraft.com/docs to the paid product alone; the site serves redirects for the slugs this rail used to push. The push script stays in the tree as history; the rail stops calling it. (cherry picked from commit 655aa13ea79e23927cde7fd47ab13b505cb042d9) --- scripts/release.sh | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 08293e3a..1a4fe575 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -248,17 +248,12 @@ else echo -e "${RED}⚠️ FORGEJO_RELEASE_TOKEN unset — no release page created; tag + CHANGELOG remain the record${NC}\n" fi -# Step 12: Push public docs to the soulcraft.com docs ingest door -# (VENUE-DOCS-RELEASE-PUSH). Skips with a loud warning when -# DOCS_INGEST_SECRET is unset; fails loudly (without undoing the publish — -# that already happened) when a push errors, so the docs site never -# silently trails npm. -echo -e "${BLUE}1️⃣2️⃣ Pushing public docs to soulcraft.com/docs...${NC}" -if node scripts/push-docs.js; then - echo -e "${GREEN}✅ Docs push step done${NC}\n" -else - echo -e "${RED}❌ Docs push FAILED — soulcraft.com/docs trails npm until re-run or interim sync${NC}\n" -fi +# Step 12 RETIRED (2026-08-31, CORTEX-SITE-BRAINY-RENAME round 12, David-ruled): +# soulcraft.com/docs carries the paid product's documentation only. This +# engine's documentation home is THIS repository — README and docs/ — and the +# site serves 301s for the slugs this rail used to push. The push script stays +# in the tree for history; the rail no longer calls it. +echo -e "${BLUE}Docs step: this engine documents itself in its own repo (site push retired 2026-08-31)${NC}" echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN}🎉 Release ${NEW_VERSION} complete!${NC}" From 0f0022b1c9abd710184d0e6ac573b6a07d8ec068 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 12:34:36 -0700 Subject: [PATCH 074/150] chore(release): 10.4.5 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a54d609e..6c05eba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31) + +- build(release): the docs-push step retires — this engine documents itself in its own repository (d6bcb14f) +- fix(generations): a sealed segment may only declare the generations it holds (a963a744) +- fix(recovery): a torn generation-log tail is a terminal verdict, never a wait (c9930871) + + ### [10.4.4](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.3...v10.4.4) (2026-08-28) - fix(vfs): the old-root sweep narrates only when it has something to say (d49148e1) diff --git a/package-lock.json b/package-lock.json index c4f66561..2528227b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.5", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 06ce0253..31448825 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.4", + "version": "10.4.5", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From 73500e7d109275570857341000c9bad44d5cc1f3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 12:59:40 -0700 Subject: [PATCH 075/150] fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit transact()'s delete legs (direct unrelate and the noun-remove cascade) hand the SAME verb object to the graph-retraction op and the metadata-retraction op. The metadata leg sanitized at PLAN time, when the verb was still clean, so the wrap returned the same reference — then the graph op's execute-time endpoint resolution (deliberately deferred for same-batch forward refs) mirrored BigInt sourceInt/targetInt onto the shared object, and the metadata op crossed the seam with them. A strict provider rightly refuses that crossing, so every transact-wrapped edge delete aborted; direct unrelate() resolves ints at build time, before its sanitize, which is why no existing gate saw it. The JSON-safe view now lives in a shared leaf (utils/jsonSafeIndexMetadata) and is applied INSIDE AddToMetadataIndexOperation and RemoveFromMetadataIndexOperation at execute and rollback time — the one place no plan-vs-execute ordering can bypass. Pins: the fleet repro, the cascade shape, a mixed batch, and unit pins that mutate the entity after construction against a strict seam (5 red before, 5 green after). --- src/brainy.ts | 30 +-- src/transaction/operations/IndexOperations.ts | 29 ++- src/utils/jsonSafeIndexMetadata.ts | 47 +++++ ...ansact-edge-delete-bigint-aliasing.test.ts | 184 ++++++++++++++++++ 4 files changed, 263 insertions(+), 27 deletions(-) create mode 100644 src/utils/jsonSafeIndexMetadata.ts create mode 100644 tests/integration/transact-edge-delete-bigint-aliasing.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index da04577e..06c947c2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -15,6 +15,7 @@ import { JsHnswVectorIndex } from './hnsw/hnswIndex.js' import { createStorage, resolveFilesystemRoot } from './storage/storageFactory.js' import type { StorageOptions } from './storage/storageFactory.js' import { rebuildCounts } from './utils/rebuildCounts.js' +import { jsonSafeIndexMetadata } from './utils/jsonSafeIndexMetadata.js' import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js' import { BaseStorage } from './storage/baseStorage.js' import { @@ -4203,32 +4204,19 @@ export class Brainy implements BrainyInterface { */ /** * @description A JSON-safe view of a record bound for the metadata-index - * crossing. The seam's metadata is JSON-safe BY CONTRACT (a native provider - * serializes it; u64 ints as Number corrupt above 2^53) — but - * {@link resolveVerbEndpointInts} MIRRORS the resolved endpoint ints onto - * the verb object itself as BigInt (`verb.sourceInt`/`targetInt`), so a - * verb object reused as index metadata carried BigInts into - * JSON.stringify, which throws, aborting the whole transaction (found by - * the first joint pair gate). Endpoint ints ride their OWN op params on the - * graph legs — the metadata crossing drops every BigInt-valued top-level - * key instead of guessing at a lossy numeric encoding. + * crossing — delegates to the shared {@link jsonSafeIndexMetadata} leaf, + * which the metadata-index transaction operations ALSO apply at execute + * and rollback time. This plan-time wrap alone proved insufficient: it + * returns the same reference when the record is clean, and `transact()`'s + * delete legs share that reference with a graph-retraction op whose + * execute-time endpoint resolution mirrors BigInt ints onto it (the full + * aliasing story lives on the leaf module's doc). * @param metadata - The candidate index-metadata record. * @returns The same object when already JSON-safe, else a shallow copy * without the BigInt-valued keys. */ private static jsonSafeIndexMetadata(metadata: unknown): unknown { - if (metadata === null || typeof metadata !== 'object') return metadata - const rec = metadata as Record - let hasBigint = false - for (const k in rec) { - if (typeof rec[k] === 'bigint') { hasBigint = true; break } - } - if (!hasBigint) return metadata - const out: Record = {} - for (const k in rec) { - if (typeof rec[k] !== 'bigint') out[k] = rec[k] - } - return out + return jsonSafeIndexMetadata(metadata) } private metadataIndexRetractionOp( diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 1bbbca88..0142dc54 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -14,6 +14,7 @@ import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' import { isZeroNormVector } from '../../utils/distance.js' +import { jsonSafeIndexMetadata } from '../../utils/jsonSafeIndexMetadata.js' import { prodLog } from '../../utils/logger.js' /** @@ -390,13 +391,21 @@ export class AddToMetadataIndexOperation implements Operation { // rollback so add + undo reference the same watermark. const generation = this.generationFn?.() - // Add to metadata index (skipFlush=true for transaction atomicity) - await this.index.addToIndex(this.id, this.entity, true, false, generation) + // The JSON-safe view is taken HERE, per crossing, never at construction: + // the entity reference this op holds can be mutated between plan and + // execute (a graph op's execute-time endpoint-int resolution mirrors + // BigInts onto a shared verb object) — see jsonSafeIndexMetadata's + // module doc. + await this.index.addToIndex( + this.id, jsonSafeIndexMetadata(this.entity), true, false, generation + ) // Return rollback action return async () => { // Remove from metadata index - await this.index.removeFromIndex(this.id, this.entity, generation) + await this.index.removeFromIndex( + this.id, jsonSafeIndexMetadata(this.entity), generation + ) } } } @@ -432,13 +441,21 @@ export class RemoveFromMetadataIndexOperation implements Operation { // Resolve the removal generation once; reuse it for the rollback re-add. const generation = this.generationFn?.() - // Remove from metadata index - await this.index.removeFromIndex(this.id, this.entity, generation) + // Sanitized per crossing, never at construction — transact()'s delete + // legs hand this op the SAME verb object the graph-retraction op's + // execute-time endpoint resolution mutates (BigInt sourceInt/targetInt), + // so a plan-time view aliases the pollution. See jsonSafeIndexMetadata's + // module doc. + await this.index.removeFromIndex( + this.id, jsonSafeIndexMetadata(this.entity), generation + ) // Return rollback action return async () => { // Re-add with original metadata (skipFlush=true) - await this.index.addToIndex(this.id, this.entity, true, false, generation) + await this.index.addToIndex( + this.id, jsonSafeIndexMetadata(this.entity), true, false, generation + ) } } } diff --git a/src/utils/jsonSafeIndexMetadata.ts b/src/utils/jsonSafeIndexMetadata.ts new file mode 100644 index 00000000..d3b1be5f --- /dev/null +++ b/src/utils/jsonSafeIndexMetadata.ts @@ -0,0 +1,47 @@ +/** + * @module utils/jsonSafeIndexMetadata + * @description The metadata-index crossing's JSON-safety law, as a leaf + * function both the coordinator and the transaction operations share. + * + * The seam's metadata is JSON-safe BY CONTRACT (a native provider serializes + * it; u64 ints as Number corrupt above 2^53) — but `resolveVerbEndpointInts` + * MIRRORS the resolved endpoint ints onto the verb object itself as BigInt + * (`verb.sourceInt`/`targetInt`), so a verb object reused as index metadata + * carries BigInts into JSON.stringify, which throws, aborting the whole + * transaction. Endpoint ints ride their OWN op params on the graph legs — the + * metadata crossing drops every BigInt-valued top-level key instead of + * guessing at a lossy numeric encoding. + * + * WHY THIS IS A LEAF MODULE, ENFORCED AT THE CROSSING: sanitizing only at + * operation-construction time is not enough. `transact()`'s delete legs pass + * the SAME verb object to both the graph-retraction op (whose endpoint-int + * thunk deliberately resolves at EXECUTE time, for same-batch forward refs) + * and the metadata-retraction op. At plan time the verb is still clean, so a + * plan-time sanitize returns the same reference — then the graph op executes + * first, mirrors the BigInt ints onto the shared object, and the metadata op + * crosses the seam with them (found by the first fleet adoption of the native + * pair: every transact-wrapped edge delete aborted). The crossing itself is + * the only place ordering cannot bypass. + */ + +/** + * A JSON-safe view of a record bound for the metadata-index crossing. + * + * @param metadata - The candidate index-metadata record. + * @returns The same object when already JSON-safe, else a shallow copy + * without the BigInt-valued keys. + */ +export function jsonSafeIndexMetadata(metadata: unknown): unknown { + if (metadata === null || typeof metadata !== 'object') return metadata + const rec = metadata as Record + let hasBigint = false + for (const k in rec) { + if (typeof rec[k] === 'bigint') { hasBigint = true; break } + } + if (!hasBigint) return metadata + const out: Record = {} + for (const k in rec) { + if (typeof rec[k] !== 'bigint') out[k] = rec[k] + } + return out +} diff --git a/tests/integration/transact-edge-delete-bigint-aliasing.test.ts b/tests/integration/transact-edge-delete-bigint-aliasing.test.ts new file mode 100644 index 00000000..b3902571 --- /dev/null +++ b/tests/integration/transact-edge-delete-bigint-aliasing.test.ts @@ -0,0 +1,184 @@ +/** + * @module tests/integration/transact-edge-delete-bigint-aliasing + * @description Regression for a fleet-adoption blocker: ANY edge delete + * inside `transact()` — a direct unrelate or a noun-remove's cascade — + * aborted with the metadata seam's BigInt JSON-guard error on a strict + * (native) metadata provider. + * + * The aliasing chain: `planTxUnrelate`/the remove-cascade pass the SAME verb + * object to the graph-retraction op and the metadata-retraction op. The + * metadata leg's JSON-safe wrap ran at PLAN time, when the verb was still + * clean — so it returned the same reference. At EXECUTE time the graph op + * runs first and `resolveVerbEndpointInts` mirrors BigInt + * `sourceInt`/`targetInt` onto the shared object (deliberately deferred for + * same-batch forward refs — see transact-forward-ref-graph.test.ts); the + * metadata op then crossed the seam with the polluted object. Direct + * `unrelate()` resolves ints at BUILD time, before its sanitize, which is why + * only the transact() shapes ever hit it. + * + * Fix under pin: the JSON-safe view is taken AT THE CROSSING — inside the + * metadata-index operations' execute/rollback — so no plan-vs-execute + * ordering can bypass it. The JS baseline index tolerates BigInts (it would + * mask the bug), so these pins SPY on the seam and assert what actually + * crossed, exactly as a strict native provider would judge it. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { + AddToMetadataIndexOperation, + RemoveFromMetadataIndexOperation +} from '../../src/transaction/operations/index.js' + +let seq = 0 +const freshId = (): string => + `00000000-0000-4000-8000-${(++seq).toString(16).padStart(12, '0')}` + +/** Top-level BigInt-valued keys of a candidate seam crossing (the guard's law). */ +const bigintKeys = (metadata: unknown): string[] => { + if (metadata === null || typeof metadata !== 'object') return [] + return Object.entries(metadata as Record) + .filter(([, v]) => typeof v === 'bigint') + .map(([k]) => k) +} + +describe('transact() edge deletes never carry BigInt across the metadata seam', () => { + let dir: string + let brain: any + let crossings: Array<{ door: string; id: string; keys: string[] }> + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-tx-bigint-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + + // Spy on the seam the way a strict native provider judges it: record the + // BigInt-valued top-level keys of every metadata argument that crosses. + // The JS baseline index tolerates BigInts, so without this the baseline + // run would green a shape the native pair aborts on. + crossings = [] + const index = brain.metadataIndex + for (const door of ['addToIndex', 'removeFromIndex'] as const) { + const real = index[door].bind(index) + index[door] = (id: string, metadata: unknown, ...rest: unknown[]) => { + crossings.push({ door, id, keys: bigintKeys(metadata) }) + return real(id, metadata, ...rest) + } + } + }) + + afterEach(async () => { + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('CASE 1 (the fleet repro): relate, then transact([{op: unrelate}])', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const verbId = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + + crossings.length = 0 + await brain.transact([{ op: 'unrelate', id: verbId }]) + + const polluted = crossings.filter((c) => c.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.storage.getVerb(verbId)).toBeFalsy() + }) + + it('CASE 2 (the cascade shape): transact([{op: remove}]) cascading edge deletes', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const c = await brain.add({ id: freshId(), data: 'c', type: NounType.Thing }) + const ab = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + const ca = await brain.relate({ from: c, to: a, type: VerbType.RelatedTo }) + + crossings.length = 0 + await brain.transact([{ op: 'remove', id: a }]) + + const polluted = crossings.filter((c2) => c2.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.get(a)).toBeFalsy() + expect(await brain.storage.getVerb(ab)).toBeFalsy() + expect(await brain.storage.getVerb(ca)).toBeFalsy() + }) + + it('CASE 3 (one batch, both legs): adds + relate + unrelate of a pre-existing edge', async () => { + const a = await brain.add({ id: freshId(), data: 'a', type: NounType.Thing }) + const b = await brain.add({ id: freshId(), data: 'b', type: NounType.Thing }) + const old = await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + + const x = freshId() + crossings.length = 0 + await brain.transact([ + { op: 'add', id: x, data: 'x', type: NounType.Thing }, + { op: 'relate', from: a, to: x, type: VerbType.RelatedTo }, + { op: 'unrelate', id: old } + ]) + + const polluted = crossings.filter((c) => c.keys.length > 0) + expect(polluted).toEqual([]) + expect(await brain.storage.getVerb(old)).toBeFalsy() + const edges = await brain.related({ from: a }) + expect(edges.length).toBe(1) + expect(edges[0].id).not.toBe(old) + }) +}) + +describe('the metadata-index operations sanitize at the crossing, not at construction', () => { + /** A strict seam: refuses BigInts exactly as the native provider does. */ + const strictIndex = () => { + const seen: Array<{ door: string; keys: string[] }> = [] + const judge = (door: string, metadata: unknown) => { + const keys = bigintKeys(metadata) + seen.push({ door, keys }) + if (keys.length > 0) { + throw new Error( + `${door}: the metadata object violates the provider seam's JSON ` + + `contract — BigInt at ${keys.join(', ')}.` + ) + } + } + return { + seen, + addToIndex: async (_id: string, metadata: unknown) => judge('addToIndex', metadata), + removeFromIndex: async (_id: string, metadata: unknown) => judge('removeFromIndex', metadata) + } + } + + it('RemoveFromMetadataIndexOperation: entity mutated AFTER construction still crosses clean', async () => { + const index = strictIndex() + const verb: Record = { id: 'v1', sourceId: 'a', targetId: 'b' } + const op = new RemoveFromMetadataIndexOperation(index as any, 'v1', verb, () => 7n) + + // The graph leg's execute-time endpoint resolution, simulated: the shared + // object is polluted between plan and execute. + verb.sourceInt = 800_000n + verb.targetInt = 800_001n + + const rollback = await op.execute() + await rollback() + expect(index.seen.map((s) => s.keys)).toEqual([[], []]) + }) + + it('AddToMetadataIndexOperation: same law on the add leg and its rollback', async () => { + const index = strictIndex() + const verb: Record = { id: 'v2', sourceId: 'a', targetId: 'b' } + const op = new AddToMetadataIndexOperation(index as any, 'v2', verb, () => 7n) + + verb.sourceInt = 800_000n + verb.targetInt = 800_001n + + const rollback = await op.execute() + await rollback() + expect(index.seen.map((s) => s.keys)).toEqual([[], []]) + }) +}) From 4014e0f12593f9c781dde0bfcf1a93b0c90d71cc Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 31 Aug 2026 14:50:45 -0700 Subject: [PATCH 076/150] chore(release): 10.4.6 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c05eba6..7154d5a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31) + +- fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d) + + ### [10.4.5](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.4...v10.4.5) (2026-08-31) - build(release): the docs-push step retires — this engine documents itself in its own repository (d6bcb14f) diff --git a/package-lock.json b/package-lock.json index 2528227b..9e573da3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.5", + "version": "10.4.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.5", + "version": "10.4.6", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 31448825..51322998 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.5", + "version": "10.4.6", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From 5e3b343a0ea6c6d162bb27aee502e35a12acdd93 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 09:32:23 -0700 Subject: [PATCH 077/150] fix(storage): counts persistence is single-flight, coalesced, and never races its own temp file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit persistCounts() was write-through on every count change with no serialization, and the atomic writer named its temp file with millisecond granularity. Two persists inside one millisecond shared the temp path: both wrote it, the first rename consumed it, the second rename found nothing — ENOENT, roughly 1,500 times a day on a busy production brain, with a full ledger write per change behind it. No data was lost (the surviving rename carried a complete ledger and the next change re-persisted), but the race was real and the write rate absurd. flushCounts() now runs exactly one persist at a time; requests arriving during it collapse into one trailing pass that carries the burst's final state — N changes cost at most two writes. writeFileAtomic() adds a per-process sequence to the temp name so no two writes can share a path. Pinned: a 25-change burst → ≤2 ledger writes, zero errors, ledger equal to memory; parallel real writes land complete; three same-instant atomic writes own three distinct temp paths. --- src/storage/adapters/baseStorageAdapter.ts | 51 ++++++-- src/storage/adapters/fileSystemStorage.ts | 9 +- .../counts-persist-single-flight.test.ts | 111 ++++++++++++++++++ 3 files changed, 162 insertions(+), 9 deletions(-) create mode 100644 tests/integration/counts-persist-single-flight.test.ts diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index cabe2e30..a90adb93 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1089,6 +1089,10 @@ export abstract class BaseStorageAdapter implements StorageAdapter { // Counts changed since the last persist? Drives the write-through flush. protected pendingCountPersist = false + /** The one persist running right now, if any (single-flight law — see flushCounts). */ + private countPersistInFlight: Promise | null = null + /** The one trailing persist a burst has queued behind the in-flight one. */ + private countPersistTrailing: Promise | null = null /** * Get total noun count - O(1) operation @@ -1341,15 +1345,46 @@ export abstract class BaseStorageAdapter implements StorageAdapter { return } - try { - // Persist to storage (implemented by subclass) - await this.persistCounts() - this.pendingCountPersist = false - } catch (error) { - console.error('CRITICAL: Failed to flush counts to storage:', error) - // Keep pending flag set so we retry on next operation - throw error + // SINGLE-FLIGHT, COALESCED. Counts are write-through on every change, so + // a burst of writes used to launch one persist per change, all in flight + // together. Two of them inside the same millisecond shared the atomic + // writer's temp path (`.tmp--`): both wrote it, the first rename + // consumed it, the second rename found nothing — ENOENT, ~1,500 times a + // day on a busy production brain, with a full ledger write per change + // behind it. Now exactly one persist runs at a time; requests that arrive + // while it runs collapse into ONE trailing persist that carries the final + // state. A burst of N changes costs at most two writes and never races + // itself. + if (this.countPersistInFlight) { + // The in-flight write may have already serialised a stale snapshot — + // ask for one more pass after it, and let every caller in this burst + // await that same pass. + if (!this.countPersistTrailing) { + this.countPersistTrailing = this.countPersistInFlight + .catch(() => undefined) + .then(() => { + this.countPersistTrailing = null + return this.flushCounts() + }) + } + return this.countPersistTrailing } + + this.countPersistInFlight = (async () => { + try { + // Persist to storage (implemented by subclass) + this.pendingCountPersist = false + await this.persistCounts() + } catch (error) { + // Keep the flag set so the next operation retries. + this.pendingCountPersist = true + console.error('CRITICAL: Failed to flush counts to storage:', error) + throw error + } finally { + this.countPersistInFlight = null + } + })() + return this.countPersistInFlight } /** diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 5ec1d88e..87b6406f 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -2400,8 +2400,15 @@ export class FileSystemStorage extends BaseStorage { * Atomic write via temp-file-then-rename so concurrent readers never see a * half-written lock JSON. Reused by writer-lock writes + heartbeat. */ + /** Monotonic per-process sequence so two atomic writes never share a temp path. */ + private static atomicWriteSeq = 0 + private async writeFileAtomic(filePath: string, contents: string): Promise { - const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}` + // pid + timestamp alone collided: two writers of the same target inside + // one millisecond shared this path, and the loser's rename found the + // winner had already moved it (ENOENT). The sequence makes every call's + // temp path its own. + const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}-${++FileSystemStorage.atomicWriteSeq}` await fs.promises.writeFile(tmp, contents) await fs.promises.rename(tmp, filePath) } diff --git a/tests/integration/counts-persist-single-flight.test.ts b/tests/integration/counts-persist-single-flight.test.ts new file mode 100644 index 00000000..5acbdcc3 --- /dev/null +++ b/tests/integration/counts-persist-single-flight.test.ts @@ -0,0 +1,111 @@ +/** + * @module tests/integration/counts-persist-single-flight + * @description Regression for a production race in FileSystemStorage's + * counts ledger: `persistCounts()` was write-through on every count change + * with no serialization, and the atomic writer named its temp file with + * millisecond granularity (`.tmp--`). Two persists inside one + * millisecond shared the temp path — both wrote it, the first rename + * consumed it, the second rename found nothing: ENOENT, ~1,500 times a day + * on a busy production brain, with a full ledger write per change behind it. + * + * Under pin: persists are single-flight and coalesced — one in flight, at + * most one trailing pass carrying the burst's final state — and every atomic + * write owns a unique temp path. A burst of N count changes costs at most + * two ledger writes, never errors, and leaves a ledger equal to memory. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +describe('counts persistence is single-flight, coalesced, and never races its own temp file', () => { + let dir: string + let brain: any + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-counts-race-')) + brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + dimensions: 384, + silent: true + }) + await brain.init() + }) + + afterEach(async () => { + vi.restoreAllMocks() + await brain.close() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('a burst of concurrent count changes → at most two ledger writes, zero errors, ledger == memory', async () => { + const storage = brain.storage + const countsPath: string = storage.countsFilePath + expect(countsPath, 'the filesystem adapter persists a counts ledger').toBeTruthy() + + // Let init's own persists settle so the burst is measured alone. + await storage.flushCounts?.() + + const renameSpy = vi.spyOn(fs.promises, 'rename') + const errorSpy = vi.spyOn(console, 'error') + + // Twenty-five concurrent count changes — the shape of a write burst; each + // used to launch its own persist. + const BURST = 25 + await Promise.all( + Array.from({ length: BURST }, () => storage.scheduleCountPersist()) + ) + + const ledgerRenames = renameSpy.mock.calls.filter(([, to]) => String(to) === countsPath) + expect(ledgerRenames.length, 'single-flight + one trailing pass').toBeLessThanOrEqual(2) + expect(ledgerRenames.length, 'the burst was persisted at all').toBeGreaterThanOrEqual(1) + + const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts')) + expect(persistErrors).toEqual([]) + + const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) + expect(ledger.totalNounCount).toBe(storage.totalNounCount) + expect(ledger.totalVerbCount).toBe(storage.totalVerbCount) + }) + + it('real writes in parallel: the ledger lands complete and no persist error is logged', async () => { + const storage = brain.storage + const countsPath: string = storage.countsFilePath + const errorSpy = vi.spyOn(console, 'error') + + await Promise.all( + Array.from({ length: 12 }, (_, i) => + brain.add({ data: `burst row ${i}`, type: NounType.Thing }) + ) + ) + await storage.flushCounts?.() + + const persistErrors = errorSpy.mock.calls.filter((args) => String(args[0]).includes('persisting counts')) + expect(persistErrors).toEqual([]) + const ledger = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) + expect(ledger.totalNounCount).toBe(storage.totalNounCount) + expect(await brain.getNounCount()).toBe(ledger.totalNounCount) + }) + + it('every atomic write owns its own temp path — two writes in one millisecond never collide', async () => { + const storage = brain.storage + const tmpNames: string[] = [] + vi.spyOn(fs.promises, 'writeFile').mockImplementation(async (p: any) => { + tmpNames.push(String(p)) + }) + vi.spyOn(fs.promises, 'rename').mockImplementation(async () => undefined) + const target = path.join(dir, 'probe.json') + await Promise.all([ + storage.writeFileAtomic(target, '{"a":1}'), + storage.writeFileAtomic(target, '{"a":2}'), + storage.writeFileAtomic(target, '{"a":3}') + ]) + const probeTmps = tmpNames.filter((n) => n.startsWith(`${target}.tmp-`)) + expect(probeTmps.length).toBe(3) + expect(new Set(probeTmps).size, 'no two writes shared a temp path').toBe(3) + }) +}) From 077cbc0b6fa346b41d7418b657cdbf852960a093 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 11:29:44 -0700 Subject: [PATCH 078/150] =?UTF-8?q?fix(find):=20connected=20finds=20are=20?= =?UTF-8?q?graph-first=20=E2=80=94=20neighbours,=20then=20the=20filter=20o?= =?UTF-8?q?ver=20those=20ids,=20then=20the=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With `connected` present, find() materialized the whole-store filtered id list, paged it, hydrated the page, and only then intersected with the neighbour set. Every such call paid O(store) for the filter and the hydration of rows that were never neighbours, and a neighbour outside the first page of the filtered STORE was silently dropped — the answer depended on the store's order and the page size. The neighbour set is now the candidate universe: resolved first from the adjacency, the metadata filter evaluated over those ids only through the provider's own evaluation (a new optional `filterIdsWithin` door on MetadataIndexProvider; the reference index implements it from its own getIdsForFilter so the two can never disagree; a provider without it is served by the whole-store answer intersected here), `orderBy` sorts the whole neighbour set before the page is cut, and the vector leg walks the neighbours as its candidate set. The text leg of a hybrid find keeps its post-intersection — it has no candidate door. Pinned in tests/integration/find-connected-order.test.ts: paging reaches every matching neighbour and never a non-neighbour; a `missing` negation is evaluated over the neighbours; the index is asked about the neighbour ids only and hydration is one page; orderBy sorts the whole set; the vector leg stays inside the neighbours; an edgeless anchor answers [] before the filter is asked. --- src/brainy.ts | 121 ++++++++++--- src/plugin.ts | 13 ++ src/utils/metadataIndex.ts | 13 ++ .../integration/find-connected-order.test.ts | 165 ++++++++++++++++++ 4 files changed, 290 insertions(+), 22 deletions(-) create mode 100644 tests/integration/find-connected-order.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 06c947c2..17fa4ad9 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -7470,7 +7470,37 @@ export class Brainy implements BrainyInterface { // JS path — there the materialized `candidateIds` restricts the walk instead. let preResolvedAllowedIds: OpaqueIdSet | undefined - if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { + // Graph-first law (10.4.8, BRAINY-PROD-LATENCY-TRIAD rounds 44/45): with + // `connected` present the NEIGHBOUR SET is the candidate universe. It is + // resolved first from the adjacency (O(neighbours)), the metadata filter + // is evaluated over those ids only, and paging happens LAST. The earlier + // order materialized the whole-store filtered id list, paged it, hydrated + // the page, and only then intersected with the neighbours — O(store) per + // call, and a neighbour outside the first page was silently dropped. + let graphFirstIds: string[] | null = null + if (hasGraphCriteria) { + graphFirstIds = await this.resolveConnectedIds(params) + if (hiddenIds.size > 0) { + graphFirstIds = graphFirstIds.filter((id) => !hiddenIds.has(id)) + } + if ( + graphFirstIds.length > 0 && + (params.where || params.type || params.subtype || params.service || params.excludeVFS) + ) { + preResolvedFilter = this.buildMetadataFilter(params) + graphFirstIds = await this.filterIdsWithinBelted(preResolvedFilter, graphFirstIds) + } + if (graphFirstIds.length === 0) { + return [] + } + if (!hasVectorSearchCriteria) { + return await this.pageConnectedIds(params, graphFirstIds) + } + // The vector leg walks ONLY the neighbours (its candidate walk). The + // filter is already applied above, so no opaque universe is produced — + // it would describe the whole store, not the neighbour set. + preResolvedMetadataIds = graphFirstIds + } else if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { preResolvedFilter = this.buildMetadataFilter(params) preResolvedMetadataIds = await this.filterIdsBelted(preResolvedFilter) @@ -7659,9 +7689,11 @@ export class Brainy implements BrainyInterface { } } - // Graph search component with O(1) traversal - if (params.connected) { - results = await this.executeGraphSearch(params, results) + // The text leg of a hybrid find has no candidate door, so its hits are + // held to the neighbour set here; the vector leg walked only the neighbours. + if (graphFirstIds !== null && results.length > 0) { + const neighbourSet = new Set(graphFirstIds) + results = results.filter((r) => neighbourSet.has(r.id)) } // Apply fusion scoring if requested @@ -12776,6 +12808,29 @@ export class Brainy implements BrainyInterface { } } + /** + * The id-scoped twin of {@link filterIdsBelted}: evaluate `filter` over `ids` + * only, through the provider's own evaluation so the answer can never drift + * from `getIdsForFilter`'s. A provider without the door is served by its + * whole-store answer intersected here (the reference index implements the + * door itself). Same belt: field refusals cross as `BrainyFieldRefusal`. + */ + private async filterIdsWithinBelted(filter: unknown, ids: readonly string[]): Promise { + this.ensureIndexesLoaded(['metadata']) + const mip = this.metadataIndex as unknown as MetadataIndexProvider + try { + if (typeof mip.filterIdsWithin === 'function') { + return await mip.filterIdsWithin(filter, ids) + } + const matched = new Set(await this.metadataIndex.getIdsForFilter(filter)) + return ids.filter((id) => matched.has(id)) + } catch (err) { + const normalized = asBrainyFieldRefusal(err) + if (normalized) throw normalized + throw err + } + } + async getIndexStatus(): Promise<{ initialized: boolean /** `true` once open()'s index-build-if-needed step has run. Named for API @@ -15759,16 +15814,16 @@ export class Brainy implements BrainyInterface { } /** - * Execute graph search component. + * Resolve `params.connected` to the neighbour id set — the graph-first + * find's candidate universe (deterministic traversal order, anchors excluded). * * Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via - * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. Previously this read - * only `from`/`to`/`direction` and did a single 1-hop `getNeighbors()`, so `depth` and `via` - * were silently ignored — `find({ connected: { from, depth: 3 } })` returned only the - * immediate neighbour at every depth. + * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. An empty set + * is re-verified against the adjacency before it is believed — a not-serving + * adjacency throws rather than answering `[]` as truth. */ - private async executeGraphSearch(params: FindParams, existingResults: Result[]): Promise[]> { - if (!params.connected) return existingResults + private async resolveConnectedIds(params: FindParams): Promise { + if (!params.connected) return [] const { from, to, depth, direction = 'both' } = params.connected const via = params.connected.via ?? params.connected.type @@ -15822,8 +15877,8 @@ export class Brainy implements BrainyInterface { if (anchorInt === undefined) return new Set() // unmapped → no relations const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType) - // No limit: match the JS BFS exactly — overall result limiting happens - // downstream against existingResults. + // No limit: match the JS BFS exactly — the page is cut downstream, + // after the metadata filter, by pageConnectedIds / the candidate walk. const reachedInts = await provider.findConnectedSubtype( anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null ) @@ -15908,22 +15963,44 @@ export class Brainy implements BrainyInterface { await this.verifyGraphAdjacencyLive() } - // Filter existing results to only connected entities - if (existingResults.length > 0) { - return existingResults.filter(r => connectedIds.has(r.id)) - } + return [...connectedIds] + } - // Batch-load connected entities for fast cloud-storage performance + /** + * Page and hydrate an already-filtered neighbour set — the pure graph (and + * graph + metadata) find's tail. `orderBy` sorts the WHOLE set by field value + * before the page is cut (never the page after), null values last on `asc` + * and first on `desc`; without `orderBy` the traversal order stands. + */ + private async pageConnectedIds(params: FindParams, ids: string[]): Promise[]> { + const limit = params.limit || 10 + const offset = params.offset || 0 + let ordered = ids + if (params.orderBy) { + const field = params.orderBy + const asc = (params.order || 'asc') === 'asc' + const valued = await Promise.all( + ids.map(async (id) => ({ id, value: await this.metadataIndex.getFieldValueForEntity(id, field) })) + ) + valued.sort((a, b) => { + if (a.value == null && b.value == null) return 0 + if (a.value == null) return asc ? 1 : -1 + if (b.value == null) return asc ? -1 : 1 + if (a.value === b.value) return 0 + const comparison = a.value < b.value ? -1 : 1 + return asc ? comparison : -comparison + }) + ordered = valued.map((v) => v.id) + } + const pageIds = ordered.slice(offset, offset + limit) + const entitiesMap = await this.batchGet(pageIds) const results: Result[] = [] - const ids = [...connectedIds] - const entitiesMap = await this.batchGet(ids) - for (const id of ids) { + for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { results.push(this.createResult(id, 1.0, entity)) } } - return results } diff --git a/src/plugin.ts b/src/plugin.ts index b1aef8e0..15b14b4e 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -411,6 +411,19 @@ export interface MetadataIndexProvider { * @returns The matching id universe as an opaque set. */ getIdSetForFilter?(filter: any): Promise + /** + * @description OPTIONAL: evaluate `filter` over `ids` ONLY and return the + * survivors in the caller's order — the door a graph-first + * `find({ connected, where })` walks. The neighbour set is the universe there, + * so the filter must cost O(|ids|) membership checks, never a whole-store + * materialization. A native index answers from its roaring filter result + * (membership by entity int); the reference index answers from its own + * `getIdsForFilter`, so the two doors can never disagree. Absent → Brainy + * intersects `getIdsForFilter`'s answer with `ids` itself (correct, O(store)). + * @param filter - The same filter shape accepted by `getIdsForFilter`. + * @param ids - The candidate ids (canonical). The answer is a subsequence. + */ + filterIdsWithin?(filter: any, ids: readonly string[]): Promise getIdsForTextQuery(query: string): Promise> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 3e0e3d17..0fd312e2 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2575,6 +2575,19 @@ export class MetadataIndexManager implements MetadataIndexProvider { /** Once-per-field flag for the fallback-degradation announcement. */ private static announcedFallbackSorts = new Set() + /** + * Evaluate `filter` over `ids` only — the graph-first find's door (the + * neighbour set filtered by id, never the store filtered and then + * intersected). This index answers from its own `getIdsForFilter`, so the + * two doors cannot disagree; the cost is that of the filter over this + * in-memory index, and the answer keeps the caller's order. + */ + async filterIdsWithin(filter: any, ids: readonly string[]): Promise { + if (ids.length === 0) return [] + const matched = new Set(await this.getIdsForFilter(filter)) + return ids.filter((id) => matched.has(id)) + } + async getSortedIdsForFilter( filter: any, orderBy: string, diff --git a/tests/integration/find-connected-order.test.ts b/tests/integration/find-connected-order.test.ts new file mode 100644 index 00000000..b04e7f99 --- /dev/null +++ b/tests/integration/find-connected-order.test.ts @@ -0,0 +1,165 @@ +/** + * @module tests/integration/find-connected-order + * @description The graph-first law for `find({ connected })` (10.4.8). + * + * With `connected` present the neighbour set is the candidate universe: it is + * resolved from the adjacency first, the metadata filter is evaluated over + * those ids only, and the page is cut last. The earlier order materialized the + * whole-store filtered id list, paged it, hydrated the page, and only then + * intersected with the neighbours — so a neighbour outside the first page of + * the filtered STORE was silently dropped, and every call paid O(store). + * + * These pins hold both halves. The answer: every matching neighbour is + * reachable by paging, a non-neighbour never appears, a negation (`missing`) + * is evaluated over the neighbours, `orderBy` sorts the whole neighbour set + * before the page is cut, and the vector leg walks the neighbours only. The + * cost shape: the metadata index is asked about the neighbour ids only, and + * hydration is one page — never the store. + */ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { v5 } from '../../src/universal/uuid' +import { generateTestVector } from '../helpers/test-factory' + +/** Matching rows that are NOT neighbours — added FIRST, so the whole-store filtered list leads with them. */ +const NOISE = 120 +/** Matching rows that ARE neighbours of the anchor. */ +const NEIGHBOURS = 30 +/** Neighbours carrying `retracted: true` — excluded by the `missing` negation. */ +const RETRACTED = 4 + +describe('find({ connected }) is graph-first: neighbours → filter → page', () => { + let brain: Brainy + const anchor = 'anchor' + const sharedVector = generateTestVector() + const neighbourIds = new Set(Array.from({ length: NEIGHBOURS }, (_, i) => v5(`nb-${i}`))) + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + await brain.add({ + id: anchor, + data: 'the anchor', + type: NounType.Person, + metadata: { kind: 'anchor' }, + vector: generateTestVector() + }) + for (let i = 0; i < NOISE; i++) { + await brain.add({ + id: `noise-${i}`, + data: `noise ${i}`, + type: NounType.Person, + metadata: { kind: 'note', rank: 1000 + i }, + vector: sharedVector + }) + } + for (let i = 0; i < NEIGHBOURS; i++) { + await brain.add({ + id: `nb-${i}`, + data: `neighbour ${i}`, + type: NounType.Person, + metadata: { kind: 'note', rank: i + 1, ...(i < RETRACTED ? { retracted: true } : {}) }, + vector: sharedVector + }) + await brain.relate({ from: anchor, to: `nb-${i}`, type: VerbType.Knows }) + } + }) + + afterAll(async () => { + brain = null as any + }) + + it('returns the matching neighbours page by page — none dropped, never a non-neighbour', async () => { + const seen = new Set() + for (let offset = 0; offset <= NEIGHBOURS; offset += 10) { + const page = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + limit: 10, + offset + }) + expect(page).toHaveLength(offset < NEIGHBOURS ? 10 : 0) + for (const r of page) { + expect(neighbourIds.has(r.entity.id)).toBe(true) + expect(seen.has(r.entity.id)).toBe(false) + seen.add(r.entity.id) + } + } + expect(seen.size).toBe(NEIGHBOURS) + }) + + it('evaluates a negation (`missing`) over the neighbour set, not the store', async () => { + const results = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note', retracted: { missing: true } }, + limit: 100 + }) + expect(results).toHaveLength(NEIGHBOURS - RETRACTED) + for (const r of results) { + expect(neighbourIds.has(r.entity.id)).toBe(true) + expect(r.entity.metadata.retracted).toBeUndefined() + } + }) + + it('asks the metadata index about the neighbour ids only, and hydrates one page', async () => { + const index = (brain as any).metadataIndex + const within = vi.spyOn(index, 'filterIdsWithin') + const hydrate = vi.spyOn(brain as any, 'batchGet') + try { + const results = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + limit: 10 + }) + expect(results).toHaveLength(10) + expect(within).toHaveBeenCalledTimes(1) + const askedIds = within.mock.calls[0][1] as string[] + expect(askedIds).toHaveLength(NEIGHBOURS) + for (const id of askedIds) expect(neighbourIds.has(id)).toBe(true) + expect(hydrate).toHaveBeenCalledTimes(1) + expect(hydrate.mock.calls[0][0]).toHaveLength(10) + } finally { + within.mockRestore() + hydrate.mockRestore() + } + }) + + it('orders the WHOLE neighbour set before cutting the page', async () => { + const results = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + orderBy: 'rank', + order: 'desc', + limit: 5 + }) + expect(results.map((r) => r.entity.metadata.rank)).toEqual([30, 29, 28, 27, 26]) + }) + + it('walks the vector leg over the neighbours only', async () => { + const results = await brain.find({ + vector: sharedVector, + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + limit: 5 + }) + expect(results).toHaveLength(5) + for (const r of results) expect(neighbourIds.has(r.entity.id)).toBe(true) + }) + + it('an anchor without neighbours answers [] before the filter is asked', async () => { + const index = (brain as any).metadataIndex + const within = vi.spyOn(index, 'filterIdsWithin') + try { + const results = await brain.find({ + connected: { from: 'noise-0', direction: 'out' }, + where: { kind: 'note' }, + limit: 10 + }) + expect(results).toEqual([]) + expect(within).not.toHaveBeenCalled() + } finally { + within.mockRestore() + } + }) +}) From e64e2bc17580737a0b7a63e2af8ea6b6c527278a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 12:04:29 -0700 Subject: [PATCH 079/150] =?UTF-8?q?docs(releases):=20the=20release-notes?= =?UTF-8?q?=20door=20=E2=80=94=20owner-language=20notes=20for=20both=20eng?= =?UTF-8?q?ines,=20backfilled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fleet's releases wall reads one public URL per product. These files are that door for Brainy and Open Brainy: newest first, honest history from the changelog, one entry appended by every release from here on. --- releases/brainy.json | 52 +++++++++++++++++++++++ releases/open-brainy.json | 87 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 releases/brainy.json create mode 100644 releases/open-brainy.json diff --git a/releases/brainy.json b/releases/brainy.json new file mode 100644 index 00000000..17217a6e --- /dev/null +++ b/releases/brainy.json @@ -0,0 +1,52 @@ +{ + "product": "brainy", + "entries": [ + { + "version": "11.0.3", + "date": "2026-09-01", + "headline": "The embedding upgrade ceremony runs on every brain", + "items": [ + "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", + "A one-fix release; nothing else changed." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.2", + "date": "2026-08-31", + "headline": "One embedding quality everywhere, 3–4× faster imports", + "items": [ + "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", + "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", + "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.1", + "date": "2026-08-31", + "headline": "Deletes inside transactions are safe", + "items": [ + "Deleting relations inside a transact() no longer corrupts index bookkeeping.", + "A store that deletes its last relation keeps serving instead of refusing." + ], + "url": null, + "thumb": null + }, + { + "version": "11.0.0", + "date": "2026-08-28", + "headline": "One install, one engine — Brainy", + "items": [ + "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", + "A missing native build refuses loudly with its cures named; nothing falls back silently.", + "Stores open in place — no migration." + ], + "url": null, + "thumb": null + } + ], + "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." +} diff --git a/releases/open-brainy.json b/releases/open-brainy.json new file mode 100644 index 00000000..21014f5b --- /dev/null +++ b/releases/open-brainy.json @@ -0,0 +1,87 @@ +{ + "product": "open-brainy", + "entries": [ + { + "version": "10.4.6", + "date": "2026-08-31", + "headline": "Transactions cross the index seam safely", + "items": [ + "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", + "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", + "thumb": null + }, + { + "version": "10.4.5", + "date": "2026-08-31", + "headline": "Recovery tells the truth, docs live at home", + "items": [ + "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", + "A sealed segment declares only the generations it actually holds.", + "The engine's documentation now publishes from its own repository." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", + "thumb": null + }, + { + "version": "10.4.4", + "date": "2026-08-28", + "headline": "Faster opens, quieter idle", + "items": [ + "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", + "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", + "A slow open now names the exact step it is in, so operators see what is being paid and why." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", + "thumb": null + }, + { + "version": "10.4.3", + "date": "2026-08-27", + "headline": "Open Brainy, under its own name", + "items": [ + "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", + "No code changes; your imports change once and everything else stays put." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", + "thumb": null + }, + { + "version": "10.4.2", + "date": "2026-08-27", + "headline": "Vectors that lie are refused, counts that drift are caught", + "items": [ + "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", + "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", + "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", + "thumb": null + }, + { + "version": "10.4.1", + "date": "2026-08-26", + "headline": "Writes that change nothing cost nothing", + "items": [ + "The read gate is per index family, and a write carrying unchanged data never re-embeds.", + "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", + "thumb": null + }, + { + "version": "10.4.0", + "date": "2026-08-26", + "headline": "Repair routing, the vector ledger, and honest empties", + "items": [ + "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", + "An empty string is real data, not a missing field.", + "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", + "thumb": null + } + ], + "history": "Earlier releases are recorded in CHANGELOG.md in this repository." +} From 88e79729d39744e35c188bca22ce0786946973d6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 12:17:55 -0700 Subject: [PATCH 080/150] perf(open): pending-embed recovery is bounded by a low-water mark and runs behind the doors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recovery fold scanned the generation log from generation 1 at every open, on the open's foreground — O(whole history) on long-lived brains (measured at two minutes of a large brain's open). Now an advisory mark records the log's head whenever the pending set drains to empty (and at clean close when empty); recovery scans from the mark + 1. The mark is advisory and monotone-safe: stale-low costs a longer scan, never a marker. The fold itself moves behind the doors as a latched background task — the embed worker starts when it settles, and awaitPendingEmbeds() and close() wait on the latch first, so no caller can observe a half-recovered set. A pending embed's outcome was always eventual; moving its recovery off the foreground changes when the worker starts, never whether a marker is honored. Pinned in tests/integration/pending-embed-low-water.test.ts: the drain writes the mark and the next open scans from mark + 1; a pending embed enqueued after the mark survives an unclean stop; open arms the fold as a background latch the barrier waits on; a clean close writes the mark even without a drain. --- src/brainy.ts | 131 ++++++++++++---- .../pending-embed-low-water.test.ts | 145 ++++++++++++++++++ 2 files changed, 249 insertions(+), 27 deletions(-) create mode 100644 tests/integration/pending-embed-low-water.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 06c947c2..c9f24873 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1820,31 +1820,31 @@ export class Brainy implements BrainyInterface { // a deferred write's ack and its background embed DELAYED a vector; // this is where it lands. if (!this.isReadOnly) { - try { - await step( - 'bridge-pending-embed-sidecars', - 'migrating any pre-log deferred-embed marker files into the generation log', - () => this.bridgeLegacyPendingEmbedSidecars() - ) - await step( - 'recover-pending-embeds', - 'folding the generation log\'s deferred-embed markers back into the pending set', - () => this.recoverPendingEmbedsFromLog() - ) - if (this._pendingEmbedIds.size > 0) { - prodLog.info( - `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + - `session — resuming in the background` + // BEHIND THE DOORS (the open pays nothing here): the bridge + the + // recovery fold run as one latched background task; the embed worker + // starts when it settles. A pending embed's outcome was always + // eventual — moving its recovery off the open's foreground changes + // when the worker starts, never whether a marker is honored. + // awaitPendingEmbeds() and close() wait on the latch first. + this._pendingEmbedRecovery = (async () => { + try { + await this.bridgeLegacyPendingEmbedSidecars() + await this.recoverPendingEmbedsFromLog() + if (this._pendingEmbedIds.size > 0) { + prodLog.info( + `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + + `session — resuming in the background` + ) + const t = setTimeout(() => this.kickEmbedWorker(), 0) + ;(t as { unref?: () => void }).unref?.() + } + } catch (err) { + prodLog.warn( + `[Brainy] pending-embed recovery failed: ${(err as Error).message} — ` + + `the log's markers remain durable; recovery retries next open` ) - const t = setTimeout(() => this.kickEmbedWorker(), 0) - ;(t as { unref?: () => void }).unref?.() } - } catch (err) { - prodLog.warn( - `[Brainy] pending-embed recovery failed: ${(err as Error).message} — ` + - `the log's markers remain durable; recovery retries next open` - ) - } + })() } // PHASE 4 of 5 — "VFS bootstrap": shutdown-hook registration, blob @@ -2408,6 +2408,19 @@ export class Brainy implements BrainyInterface { */ private static readonly PENDING_EMBED_PREFIX = '_system/pending_embeds/' + /** + * Storage-root-relative path of the ADVISORY pending-embed low-water mark: + * `{ generation, writtenAt }`, written whenever the pending set drains to + * empty (and at clean close when empty). Every marker in facts at or below + * `generation` is consumed, so recovery scans from `generation + 1`. The + * mark is advisory and monotone-safe: stale-low costs a longer scan, never + * a lost marker; it is never required for correctness. + */ + private static readonly PENDING_EMBED_LOWWATER_PATH = '_system/pending_embeds_lowwater.json' + + /** Resolves when the background pending-embed recovery fold has settled (open arms it). */ + private _pendingEmbedRecovery: Promise | null = null + /** * @description Mark a deferred embed pending (MT5): the id joins the * in-memory fast-path set and the returned `embed.pending` record is @@ -2435,6 +2448,40 @@ export class Brainy implements BrainyInterface { */ private clearPendingEmbed(id: string): void { this._pendingEmbedIds.delete(id) + if (this._pendingEmbedIds.size === 0) this.maybeWriteEmbedLowWater() + } + + /** + * @description Advance the advisory low-water mark: called at drain-to-empty + * (and at clean close when empty), it records the fact log's CURRENT head — + * with the set empty, every marker at or below the head has been consumed, + * so the next open's recovery fold scans only what comes after. Fire-and- + * forget at the drain (close() awaits the core); loud on failure: a missed + * write costs the next open a longer scan, never a marker. No-op without a + * fact log (no durable markers exist there) and on read-only opens. + */ + private maybeWriteEmbedLowWater(): void { + void this.writeEmbedLowWater() + } + + /** The awaitable core of {@link maybeWriteEmbedLowWater} — close() awaits it. */ + private async writeEmbedLowWater(): Promise { + if (this.isReadOnly) return + const log = this.generationStore ? this.generationStore.getFactLog() : null + if (!log) return + const generation = log.headGeneration() + if (!(generation > 0)) return + try { + await this.storage.writeRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH, { + generation, + writtenAt: Date.now() + }) + } catch (err) { + prodLog.warn( + `[Brainy] pending-embed low-water write failed at generation ${generation}: ` + + `${(err as Error).message} — the next open scans from the previous mark` + ) + } } /** @@ -2445,9 +2492,14 @@ export class Brainy implements BrainyInterface { * survives the fold is exactly the set of acknowledged deferred writes * whose vectors have not landed. * - * BOUND (honest): no durable low-water mark exists for the earliest - * unconsumed pending, so the fold scans the log's committed facts from - * generation 1 — a sequential read of the log at open, O(log bytes). + * BOUND: the scan starts at the advisory low-water mark + * ({@link Brainy.PENDING_EMBED_LOWWATER_PATH}) — the log head at which the + * pending set last drained to empty — so a settled brain reads only the + * facts since then, not its whole history. Without a mark (first open + * after upgrade) it scans from generation 1, once; a stale-low mark costs + * a longer scan, never a marker. The fold runs BEHIND the doors (open + * arms it as a background task and the embed worker starts when it + * settles); {@link awaitPendingEmbeds} and close() wait for it first. * It is SKIPPED WHOLESALE when the log has never had a v2 tail * ({@link FactLog.hasV2History} — v1 facts cannot carry marker records), * so pre-cutover brains pay nothing; on a mixed log the scan still reads @@ -2460,7 +2512,18 @@ export class Brainy implements BrainyInterface { private async recoverPendingEmbedsFromLog(): Promise { const log = this.generationStore.getFactLog() if (!log || !log.hasV2History()) return - const scan = log.scanFacts({ fromGeneration: 1 }) + let fromGeneration = 1 + try { + const mark = (await this.storage.readRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH)) as { + generation?: number + } | null + if (mark && typeof mark.generation === 'number' && mark.generation > 0) { + fromGeneration = mark.generation + 1 + } + } catch { + // No mark (or unreadable): scan from 1 — correctness over cost. + } + const scan = log.scanFacts({ fromGeneration }) for await (const batch of scan.batches()) { for (const fact of batch.facts) { for (const record of fact.records ?? []) { @@ -2647,6 +2710,7 @@ export class Brainy implements BrainyInterface { * before I proceed" callers use this; nothing else ever needs to wait. */ public async awaitPendingEmbeds(): Promise { + if (this._pendingEmbedRecovery) await this._pendingEmbedRecovery while (this._pendingEmbedIds.size > 0 || this._embedWorkerFlight) { this.kickEmbedWorker() await (this._embedWorkerFlight ?? Promise.resolve()) @@ -19443,6 +19507,19 @@ export class Brainy implements BrainyInterface { * terminal releases have run. */ async close(): Promise { + if (this._pendingEmbedRecovery) { + // Settle the background marker fold before the durable steps — its scan + // is bounded by the low-water mark (a full scan happens at most once, + // on the first open after upgrade). + const settleStart = Date.now() + await this._pendingEmbedRecovery + const settleMs = Date.now() - settleStart + if (settleMs >= 1000) { + prodLog.info(`[Brainy] close: pending-embed recovery settled in ${settleMs}ms`) + } + this._pendingEmbedRecovery = null + } + if (this._pendingEmbedIds.size === 0) await this.writeEmbedLowWater() let closeFailure: unknown = null try { await this.closeDurableSteps() diff --git a/tests/integration/pending-embed-low-water.test.ts b/tests/integration/pending-embed-low-water.test.ts new file mode 100644 index 00000000..ff01b349 --- /dev/null +++ b/tests/integration/pending-embed-low-water.test.ts @@ -0,0 +1,145 @@ +/** + * @module tests/integration/pending-embed-low-water + * @description The pending-embed recovery fold is bounded and background (10.4.9). + * + * The fold used to scan the generation log from generation 1 at EVERY open, + * on the open's foreground — O(whole history) per open on long-lived brains. + * Now: an advisory low-water mark (`_system/pending_embeds_lowwater.json`) + * records the committed generation whenever the pending set drains to empty, + * recovery scans from `mark + 1`, and the fold runs behind the doors as a + * latched background task the worker, `awaitPendingEmbeds()` and `close()` + * wait on. The mark is advisory: stale-low costs a longer scan, never a + * marker — a pending embed enqueued before a crash is still recovered. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy' +import { NounType } from '../../src/types/graphTypes' + +const LOWWATER_PATH = '_system/pending_embeds_lowwater.json' + +describe('pending-embed recovery: bounded by the low-water mark, behind the doors', () => { + const roots: string[] = [] + const dir = (): string => { + const d = mkdtempSync(join(tmpdir(), 'brainy-lowwater-')) + roots.push(d) + return d + } + const open = async (root: string): Promise> => { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: root } + }) + await brain.init() + return brain + } + + afterEach(() => { + for (const d of roots.splice(0)) rmSync(d, { recursive: true, force: true }) + }) + + it('drain-to-empty writes the mark, and the next open scans from mark + 1', async () => { + const root = dir() + const brain = await open(root) + // Hold the worker so the pending state is observable, then release it. + const realKick = (brain as any).kickEmbedWorker.bind(brain) + ;(brain as any).kickEmbedWorker = () => {} + await brain.add({ + id: 'row-1', + data: 'the first deferred row', + type: NounType.Thing, + deferEmbedding: true + }) + expect(brain.pendingEmbedCount()).toBeGreaterThan(0) + ;(brain as any).kickEmbedWorker = realKick + await brain.awaitPendingEmbeds() + // The drain wrote the advisory mark (fire-and-forget: settle the microtask). + await new Promise((r) => setTimeout(r, 50)) + const mark = (await (brain as any).storage.readRawObject(LOWWATER_PATH)) as { + generation: number + } | null + expect(mark).not.toBeNull() + expect(mark!.generation).toBeGreaterThan(0) + await brain.close() + + const brain2 = await open(root) + const log = (brain2 as any).generationStore.getFactLog() + const scanSpy = vi.spyOn(log, 'scanFacts') + try { + await (brain2 as any).recoverPendingEmbedsFromLog() + expect(scanSpy).toHaveBeenCalledTimes(1) + const opts = scanSpy.mock.calls[0][0] as { fromGeneration?: number } + expect(opts.fromGeneration).toBeGreaterThanOrEqual(mark!.generation + 1) + } finally { + scanSpy.mockRestore() + await brain2.close() + } + }) + + it('a pending embed enqueued after the mark survives an unclean stop', async () => { + const root = dir() + const brain = await open(root) + await brain.add({ id: 'settled', data: 'lands before the mark', type: NounType.Thing }) + await brain.awaitPendingEmbeds() + await new Promise((r) => setTimeout(r, 50)) + + // A deferred write whose embed never lands: block the worker, then drop + // the instance without close() — the unclean-stop shape. + ;(brain as any).kickEmbedWorker = () => {} + await brain.add({ + id: 'orphan', + data: 'enqueued then abandoned', + type: NounType.Thing, + deferEmbedding: true + }) + expect(brain.pendingEmbedCount()).toBeGreaterThan(0) + // No close(): simulate the crash by releasing only the writer lock so the + // next open can proceed. + await (brain as any).storage.releaseWriterLock() + + const brain2 = await open(root) + await (brain2 as any)._pendingEmbedRecovery + expect(brain2.pendingEmbedCount()).toBeGreaterThan(0) + await brain2.awaitPendingEmbeds() + expect(brain2.pendingEmbedCount()).toBe(0) + await brain2.close() + // Reap the crashed instance: its fence is gone, so close() fails loudly — + // swallow that here; the point is clearing its watchers and registry entry. + await brain.close().catch(() => undefined) + }) + + it('open arms the fold as a background latch; awaitPendingEmbeds waits on it', async () => { + const root = dir() + const brain = await open(root) + await brain.add({ id: 'a-row', data: 'some data', type: NounType.Thing }) + await brain.awaitPendingEmbeds() + await brain.close() + + const brain2 = await open(root) + // The latch exists the moment init() returns (writable filesystem brain)… + expect((brain2 as any)._pendingEmbedRecovery).not.toBeNull() + // …and the barrier settles it before answering. + await brain2.awaitPendingEmbeds() + expect(brain2.pendingEmbedCount()).toBe(0) + await brain2.close() + }) + + it('a clean close with an empty set writes the mark even if no drain happened', async () => { + const root = dir() + const brain = await open(root) + await brain.add({ id: 'r1', data: 'row one', type: NounType.Thing }) + await brain.awaitPendingEmbeds() + await brain.close() + // Read the mark back through the storage door (the adapter owns the + // on-disk encoding), on a fresh instance. + const brain2 = await open(root) + const mark = (await (brain2 as any).storage.readRawObject(LOWWATER_PATH)) as { + generation: number + } | null + expect(mark).not.toBeNull() + expect(mark!.generation).toBeGreaterThan(0) + await brain2.close() + }) +}) From 6a89adc46855e6d8b3ed0241fc376f7ab2ecd934 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 12:20:08 -0700 Subject: [PATCH 081/150] fix(graph): the verb fast paths honour every requested type, source, and target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit related() with a verb-type ARRAY returned edges for only the first type — the storage fast paths collapsed `verbType` (and, in their sibling blocks, `sourceId` and `targetId`) arrays to their first element, silently dropping the rest of the ask. Every consumer passing a verb list under-traversed with no error and no narration: the same quiet-loss class as the graph-first paging defect, one seam over. All four fast paths now union over the full requested set, deduped by edge id, before the metadata filters and pagination run. Pinned in tests/integration/related-verb-array.test.ts: the second requested type's edge returns in both array orders, on the anchor side, the target side, and the type-only path; a one-element array equals the scalar; no duplicates on overlap; pagination walks the union consistently. --- src/storage/baseStorage.ts | 113 ++++++++++++------- tests/integration/related-verb-array.test.ts | 89 +++++++++++++++ 2 files changed, 163 insertions(+), 39 deletions(-) create mode 100644 tests/integration/related-verb-array.test.ts diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index d8bcb780..a1cc2e35 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -2942,19 +2942,33 @@ export abstract class BaseStorage extends BaseStorageAdapter { !options.filter.service && !options.filter.metadata ) { - const sourceId = Array.isArray(options.filter.sourceId) - ? options.filter.sourceId[0] - : options.filter.sourceId + const sourceIds = Array.isArray(options.filter.sourceId) + ? options.filter.sourceId + : [options.filter.sourceId] - const verbType = Array.isArray(options.filter.verbType) - ? options.filter.verbType[0] - : options.filter.verbType + // EVERY requested verb type is honoured — an array used to collapse to + // its first element here, silently dropping the rest of the ask. + const verbTypes = new Set( + Array.isArray(options.filter.verbType) + ? options.filter.verbType + : [options.filter.verbType] + ) - // Get verbs by source, then filter by type (O(1) graph lookup + O(n) type filter), - // then apply the subtype / visibility metadata filters on the candidate set. - const verbsBySource = await this.getVerbsBySource_internal(sourceId) + // Get verbs by source (union over every requested source), filter by the + // requested type SET (O(1) graph lookup + O(n) type filter), then apply + // the subtype / visibility metadata filters on the candidate set. + const bySource: HNSWVerbWithMetadata[] = [] + const seenVerbIds = new Set() + for (const oneSource of sourceIds) { + for (const v of await this.getVerbsBySource_internal(oneSource)) { + if (!seenVerbIds.has(v.id)) { + seenVerbIds.add(v.id) + bySource.push(v) + } + } + } const filteredVerbs = this.applyVerbMetadataFilters( - verbsBySource.filter(v => v.verb === verbType), + bySource.filter(v => verbTypes.has(v.verb)), options.filter ) @@ -2985,16 +2999,22 @@ export abstract class BaseStorage extends BaseStorageAdapter { !options.filter.service && !options.filter.metadata ) { - const sourceId = Array.isArray(options.filter.sourceId) - ? options.filter.sourceId[0] - : options.filter.sourceId - - // Get verbs by source directly (hydrated with metadata), then apply the - // subtype / visibility metadata filters on the O(degree) candidate set. - const verbsBySource = this.applyVerbMetadataFilters( - await this.getVerbsBySource_internal(sourceId), - options.filter - ) + // EVERY requested source is honoured — an array used to collapse to + // its first element here, silently dropping the rest of the ask. + const onlySourceIds = Array.isArray(options.filter.sourceId) + ? options.filter.sourceId + : [options.filter.sourceId] + const sourceUnion: HNSWVerbWithMetadata[] = [] + const seenSourceVerbIds = new Set() + for (const oneSource of onlySourceIds) { + for (const v of await this.getVerbsBySource_internal(oneSource)) { + if (!seenSourceVerbIds.has(v.id)) { + seenSourceVerbIds.add(v.id) + sourceUnion.push(v) + } + } + } + const verbsBySource = this.applyVerbMetadataFilters(sourceUnion, options.filter) // Apply pagination const paginatedVerbs = verbsBySource.slice(offset, offset + limit) @@ -3023,16 +3043,22 @@ export abstract class BaseStorage extends BaseStorageAdapter { !options.filter.service && !options.filter.metadata ) { - const targetId = Array.isArray(options.filter.targetId) - ? options.filter.targetId[0] - : options.filter.targetId - - // Get verbs by target directly (hydrated with metadata), then apply the - // subtype / visibility metadata filters on the O(degree) candidate set. - const verbsByTarget = this.applyVerbMetadataFilters( - await this.getVerbsByTarget_internal(targetId), - options.filter - ) + // EVERY requested target is honoured — an array used to collapse to + // its first element here, silently dropping the rest of the ask. + const onlyTargetIds = Array.isArray(options.filter.targetId) + ? options.filter.targetId + : [options.filter.targetId] + const targetUnion: HNSWVerbWithMetadata[] = [] + const seenTargetVerbIds = new Set() + for (const oneTarget of onlyTargetIds) { + for (const v of await this.getVerbsByTarget_internal(oneTarget)) { + if (!seenTargetVerbIds.has(v.id)) { + seenTargetVerbIds.add(v.id) + targetUnion.push(v) + } + } + } + const verbsByTarget = this.applyVerbMetadataFilters(targetUnion, options.filter) // Apply pagination const paginatedVerbs = verbsByTarget.slice(offset, offset + limit) @@ -3061,16 +3087,25 @@ export abstract class BaseStorage extends BaseStorageAdapter { !options.filter.service && !options.filter.metadata ) { - const verbType = Array.isArray(options.filter.verbType) - ? options.filter.verbType[0] - : options.filter.verbType + // EVERY requested verb type is honoured — an array used to collapse to + // its first element here, silently dropping the rest of the ask. + const verbTypes = Array.isArray(options.filter.verbType) + ? options.filter.verbType + : [options.filter.verbType] - // Get verbs by type directly (hydrated with metadata), then apply the - // subtype / visibility metadata filters on the candidate set. - const verbsByType = this.applyVerbMetadataFilters( - await this.getVerbsByType_internal(verbType), - options.filter - ) + // Get verbs by each requested type (hydrated with metadata), deduped by + // id, then apply the subtype / visibility metadata filters on the set. + const byType: HNSWVerbWithMetadata[] = [] + const seenTypeVerbIds = new Set() + for (const oneType of verbTypes) { + for (const v of await this.getVerbsByType_internal(oneType)) { + if (!seenTypeVerbIds.has(v.id)) { + seenTypeVerbIds.add(v.id) + byType.push(v) + } + } + } + const verbsByType = this.applyVerbMetadataFilters(byType, options.filter) // Apply pagination const paginatedVerbs = verbsByType.slice(offset, offset + limit) diff --git a/tests/integration/related-verb-array.test.ts b/tests/integration/related-verb-array.test.ts new file mode 100644 index 00000000..36a49850 --- /dev/null +++ b/tests/integration/related-verb-array.test.ts @@ -0,0 +1,89 @@ +/** + * @module tests/integration/related-verb-array + * @description related() honours EVERY verb type in an array (10.4.9). + * + * The storage fast paths for `sourceId + verbType` and `verbType` collapsed a + * verb-type ARRAY to its first element — `related({ from, type: [a, b] })` + * silently returned only `a` edges, whichever order the array came in. The + * same quiet-loss class as the graph-first paging defect, one seam over. + * These pins seed a store where the SECOND requested type's edge must come + * back, on every path the collapse lived in. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { v5 } from '../../src/universal/uuid' + +describe('related() with a verb-type array returns every requested type', () => { + let brain: Brainy + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + for (const id of ['a', 'b', 'c', 'd']) { + await brain.add({ id, data: `node ${id}`, type: NounType.Person }) + } + await brain.relate({ from: 'a', to: 'b', type: VerbType.Supports }) + await brain.relate({ from: 'a', to: 'c', type: VerbType.RelatedTo }) + await brain.relate({ from: 'a', to: 'd', type: VerbType.Knows }) + await brain.relate({ from: 'b', to: 'c', type: VerbType.RelatedTo }) + }) + + afterAll(async () => { + brain = null as any + }) + + it('from + type array: the second type\'s edge comes back, both orders', async () => { + for (const types of [ + [VerbType.Supports, VerbType.RelatedTo], + [VerbType.RelatedTo, VerbType.Supports] + ]) { + const edges = await brain.related({ from: 'a', type: types }) + const targets = new Set(edges.map((e) => e.to)) + expect(targets.has(v5('b')), `types [${types}] missing Supports edge`).toBe(true) + expect(targets.has(v5('c')), `types [${types}] missing RelatedTo edge`).toBe(true) + expect(targets.has(v5('d'))).toBe(false) + expect(edges).toHaveLength(2) + } + }) + + it('a single-element array behaves exactly like the scalar', async () => { + const scalar = await brain.related({ from: 'a', type: VerbType.Supports }) + const array = await brain.related({ from: 'a', type: [VerbType.Supports] }) + expect(array.map((e) => e.id).sort()).toEqual(scalar.map((e) => e.id).sort()) + expect(array).toHaveLength(1) + }) + + it('no duplicate edges when types overlap the same edge set', async () => { + const edges = await brain.related({ + from: 'a', + type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows] + }) + const ids = edges.map((e) => e.id) + expect(new Set(ids).size).toBe(ids.length) + expect(edges).toHaveLength(3) + }) + + it('type-only asks (no anchor) honour the whole array too', async () => { + const edges = await brain.related({ type: [VerbType.Supports, VerbType.Knows] }) + const verbs = new Set(edges.map((e) => e.type)) + expect(verbs.has(VerbType.Supports)).toBe(true) + expect(verbs.has(VerbType.Knows)).toBe(true) + expect(edges).toHaveLength(2) + }) + + it('to + type array: the target side honours every type too', async () => { + const edges = await brain.related({ to: 'c', type: [VerbType.RelatedTo, VerbType.Supports] }) + const froms = new Set(edges.map((e) => e.from)) + expect(froms.has(v5('a'))).toBe(true) + expect(froms.has(v5('b'))).toBe(true) + expect(edges).toHaveLength(2) + }) + + it('pagination stays consistent across the union', async () => { + const page1 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2 }) + const page2 = await brain.related({ from: 'a', type: [VerbType.Supports, VerbType.RelatedTo, VerbType.Knows], limit: 2, offset: 2 }) + const all = [...page1, ...page2].map((e) => e.id) + expect(new Set(all).size).toBe(3) + }) +}) From 3e60aded36cdecd1fbb6389cbcdb394188b40bec Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 12:48:38 -0700 Subject: [PATCH 082/150] perf(vfs): repairContainment's reconcile is one paged edge walk, not one graph call per file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass 2 issued one awaited related({ to }) per VFS entity — O(entities) serialized graph calls, measured in whole minutes on large brains. Now a single paged walk over every Contains edge (type-only, 1,000 per page) feeds an in-memory group-by-target, and only actual defects mutate. The verdicts are unchanged: a stale parent's edge is removed, a missing edge is restored, duplicates cannot survive, and user knowledge edges are never touched. Pinned in tests/integration/vfs-containment-batched.test.ts: exact removed/restored counts on a seeded defect tree, tree correctness after the repair, user edges untouched, and the cost shape — related() call count independent of the entity count. --- src/vfs/VirtualFileSystem.ts | 27 +++- .../vfs-containment-batched.test.ts | 115 ++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 tests/integration/vfs-containment-batched.test.ts diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 46c6a12d..1a4b9fa5 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -2295,6 +2295,31 @@ export class VirtualFileSystem implements IVirtualFileSystem { cursor = page.nextCursor } + // Pass 2: ONE paged walk over every Contains edge, grouped by target in + // memory. The earlier shape issued one awaited related({ to }) per VFS + // entity — O(entities) serialized graph calls, measured in whole minutes + // on large brains. This shape is O(edges / page) calls regardless of how + // many entities exist; mutations alone stay per-defect. + const incomingByTarget = new Map[]>() + { + const pageSize = 1000 + let pageOffset = 0 + for (;;) { + const page = await this.brain.related({ + type: VerbType.Contains, + limit: pageSize, + offset: pageOffset + }) + for (const edge of page) { + const bucket = incomingByTarget.get(edge.to) + if (bucket) bucket.push(edge) + else incomingByTarget.set(edge.to, [edge]) + } + if (page.length < pageSize) break + pageOffset += pageSize + } + } + let removed = 0 let restored = 0 for (const { id, path } of vfsEntities) { @@ -2307,7 +2332,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { continue } - const incoming = await this.brain.related({ to: id, type: VerbType.Contains }) + const incoming = incomingByTarget.get(id) ?? [] let expectedSeen = false for (const edge of incoming) { const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true diff --git a/tests/integration/vfs-containment-batched.test.ts b/tests/integration/vfs-containment-batched.test.ts new file mode 100644 index 00000000..0a7919bf --- /dev/null +++ b/tests/integration/vfs-containment-batched.test.ts @@ -0,0 +1,115 @@ +/** + * @module tests/integration/vfs-containment-batched + * @description repairContainment costs O(edges/page) graph calls, not O(entities) (10.4.9 train). + * + * Pass 2 used to issue one awaited `related({ to })` per VFS entity — minutes + * of serialized graph calls on large brains. Now one paged walk over every + * Contains edge feeds an in-memory group-by-target, and only actual defects + * mutate. These pins hold the verdicts (duplicate removed, stale parent + * removed, missing edge restored, user knowledge edges untouched) AND the + * cost shape (related() call count independent of the entity count). + */ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' + +const FILES = 60 + +describe('repairContainment: batched pass 2', () => { + let brain: Brainy + let result: { removed: number; restored: number } + let relatedCalls = 0 + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + const vfs = (brain as any).vfs ?? (brain as any)._vfs + expect(vfs).toBeTruthy() + await vfs.init() + + // A directory and FILES entries under it, wired as real VFS rows. + const mkNode = async (id: string, path: string, vfsType: string): Promise => { + await brain.add({ + id, + data: `vfs node ${path}`, + type: NounType.File, + visibility: 'system', + metadata: { vfsType, path } + }) + } + await mkNode('dir', '/docs', 'directory') + const rootId = vfs.rootEntityId ?? (await vfs.initializeRoot?.()) + if (rootId) { + await brain.relate({ + from: rootId, + to: 'dir', + type: VerbType.Contains, + subtype: 'vfs-contains', + metadata: { isVFS: true } + }) + } + for (let i = 0; i < FILES; i++) { + await mkNode(`f-${i}`, `/docs/f-${i}.md`, 'file') + if (i === 0) continue // f-0: MISSING edge — must be restored + await brain.relate({ + from: 'dir', + to: `f-${i}`, + type: VerbType.Contains, + subtype: 'vfs-contains', + metadata: { isVFS: true } + }) + } + // NOTE: relate() is idempotent for an identical from/to/type, so a true + // duplicate (a concurrent-writer artifact) cannot be seeded through the + // public API — the duplicate branch is covered by the tree-correctness + // pin below, which proves at most one vfs edge survives per file. + // f-2: STALE parent edge (from a sibling file) — must be removed. + await brain.relate({ + from: 'f-3', + to: 'f-2', + type: VerbType.Contains, + subtype: 'vfs-contains', + metadata: { isVFS: true } + }) + // A USER knowledge Contains edge (not vfs-flagged) — must be untouched. + await brain.relate({ from: 'f-4', to: 'f-5', type: VerbType.Contains }) + + const spy = vi.spyOn(brain, 'related') + result = await vfs.repairContainment() + relatedCalls = spy.mock.calls.length + spy.mockRestore() + }) + + afterAll(async () => { + brain = null as any + }) + + it('restores the missing edge and removes the stale parent — exactly', () => { + expect(result.restored).toBe(1) // f-0's missing edge + expect(result.removed).toBe(1) // f-2's stale parent (f-3 → f-2) + }) + + it('the repaired tree is correct: every file has exactly one vfs edge from its dir', async () => { + for (let i = 0; i < 6; i++) { + const incoming = await brain.related({ to: `f-${i}`, type: VerbType.Contains }) + const vfsEdges = incoming.filter( + (e) => e.subtype === 'vfs-contains' || (e.metadata as any)?.isVFS === true + ) + expect(vfsEdges, `f-${i}`).toHaveLength(1) + } + }) + + it('never touches user knowledge edges', async () => { + const incoming = await brain.related({ to: 'f-5', type: VerbType.Contains }) + const user = incoming.filter( + (e) => e.subtype !== 'vfs-contains' && (e.metadata as any)?.isVFS !== true + ) + expect(user).toHaveLength(1) + }) + + it('cost shape: related() calls do not scale with the entity count', () => { + // One paged type-only walk (~E/1000 pages) — with 60+ entities the old + // shape issued 60+ calls; the new one a handful. Bound generously. + expect(relatedCalls).toBeLessThanOrEqual(5) + }) +}) From 4d5f823f47d924d29fd244769cadb2b0539ae9a3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 13:11:16 -0700 Subject: [PATCH 083/150] =?UTF-8?q?feat(plugin):=20an=20optional=20planFin?= =?UTF-8?q?dPage=20door=20=E2=80=94=20an=20index=20that=20can=20plan=20a?= =?UTF-8?q?=20find=20answers=20it=20in=20one=20call?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provider's read doors each serve one stage, so a find that consults three of them crosses into the index three times and marshals a result set at every crossing: a filter matching a hundred thousand rows builds a hundred thousand id strings to return a page of twenty-five. An index able to decide the stage order itself can answer the page in one call and build ids only for the page. planFindPage is optional and additive, in the shape filterIdsWithin and getIdSetForFilter already set. The hook sits above the branch selection, because the branches are what decide stage order per call site and an index that plans has to be asked before that choice is made. Absent — as it is on this engine's own index — every find is served by the stage doors exactly as before, which is what keeps this engine the ordering oracle for any index that implements one. The contract the door must keep, written where an implementer will read it: identical rows in identical order to what the stage doors would produce; the graph-first law (neighbours are the candidate universe, the filter runs over those ids, orderBy sorts the whole set, the page is cut last); null returned BEFORE any work rather than instead of an answer; and emptyAt naming the stage that produced an empty page, so the serving law is applied to the right index — an empty graph answer is re-verified against the adjacency before it is believed, and a filter-empty is not. Pinned in tests/integration/find-planner-door.test.ts: absent changes nothing; present it is asked first with normalized params, the hidden ids and the graph provider; its page is used and hydrated in its order; a declining door leaves the result identical to the no-door path; and the two emptyAt branches verify the adjacency, or correctly do not. --- src/brainy.ts | 41 ++++++ src/neural/embeddedTypeEmbeddings.ts | 4 +- src/plugin.ts | 46 +++++++ tests/integration/find-planner-door.test.ts | 137 ++++++++++++++++++++ 4 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 tests/integration/find-planner-door.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 696a5f87..464f689b 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -7344,6 +7344,47 @@ export class Brainy implements BrainyInterface { await this.verifyMetadataLive() } + // PLANNED FIND (optional provider door, `MetadataIndexProvider.planFindPage`). + // + // The stage doors below each serve one stage, so a find that consults + // three of them crosses into the index three times and marshals a result + // set at every crossing — a filter matching a hundred thousand rows + // builds a hundred thousand id strings to return a page of twenty-five. + // An index that can decide the stage order itself answers the page in one + // call and materializes ids only for the page. + // + // The hook sits ABOVE the branch selection because the branches are what + // decide stage order per call site; an index that plans has to be asked + // before that choice is made, not inside one of its arms. + // + // Optional and additive: a provider without the door, and any shape the + // door hands back, take exactly the path they always took. `null` is a + // routing decision the door must make BEFORE doing any work — never a + // partial answer. Every guard above still ran (readiness, the migration + // gate, the where-clause validation, the metadata cold-read guard), and + // the serving law is applied here on the way out: an empty answer is + // re-verified against the index that produced it before it is believed. + const planningIndex = this.metadataIndex as unknown as MetadataIndexProvider + if (typeof planningIndex.planFindPage === 'function') { + const planned = await planningIndex.planFindPage(params, [...hiddenIds], this.graphIndex) + if (planned !== null && planned !== undefined) { + if (planned.ids.length === 0) { + // A cold adjacency can report a size yet hold no edges, so an empty + // graph answer is not truth until the adjacency verifies live. A + // genuinely edgeless anchor verifies and the empty result stands. + if (planned.emptyAt === 'graph') await this.verifyGraphAdjacencyLive() + return [] + } + const plannedEntities = await this.batchGet(planned.ids) + const plannedResults: Result[] = [] + for (const id of planned.ids) { + const entity = plannedEntities.get(id) + if (entity) plannedResults.push(this.createResult(id, 1.0, entity)) + } + return plannedResults + } + } + // Handle metadata-only queries (no vector search needed) if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) { // Build filter for metadata index diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index 5b10116c..f4cdd632 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-06-29T10:04:19-07:00 + * Generated: 2026-08-27T09:18:45-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-06-29T10:04:19-07:00", + generatedAt: "2026-08-27T09:18:45-07:00", sizeBytes: { embeddings: 259584, base64: 346112 diff --git a/src/plugin.ts b/src/plugin.ts index 15b14b4e..fdad42f0 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -424,6 +424,52 @@ export interface MetadataIndexProvider { * @param ids - The candidate ids (canonical). The answer is a subsequence. */ filterIdsWithin?(filter: any, ids: readonly string[]): Promise + /** + * @description OPTIONAL: plan and execute a WHOLE `find()` — the graph + * traversal, the metadata filter, the ordering and the page — and answer the + * page's ids, or `null` for a shape this index does not plan. + * + * The doors above each serve one stage, so a `find()` that consults three of + * them crosses into the index three times and marshals a result set at every + * crossing. An index that can decide the stage ORDER itself does the whole + * thing in one call and materializes ids only for the page — a filter + * matching a hundred thousand rows then builds twenty-five id strings instead + * of a hundred thousand. + * + * The contract this door must keep, because Brainy cannot check it: + * + * - **The same answer.** Identical rows, in identical order, to what the + * stage doors would have produced for the same params. This door changes + * which code runs, never what the answer is. + * - **The law of the stages** (`find({ connected })` is graph-first): the + * neighbour set is the candidate universe, the filter is evaluated over + * those ids only, `orderBy` sorts the whole candidate set, and the page is + * cut LAST. + * - **`null` before work, not instead of an answer.** A shape the index does + * not plan must be handed back BEFORE any evaluation, so Brainy serves it + * through the stage doors exactly as it always has. Returning `null` after + * partial work, or an empty page for a shape it could not evaluate, is a + * silent wrong answer. + * - **`emptyAt` names the stage** that produced an empty page — `'graph'`, + * `'filter'`, `'visibility'` or `'none'` — so Brainy can apply its serving + * law to the right index. An empty answer from an index that is not + * serving must refuse loudly, and Brainy can only re-verify what it is told. + * + * Absent → every `find()` is served by the stage doors, which is Brainy's + * own behaviour and the ordering oracle for any implementation of this one. + * @param params - The find params, already normalized by `find()` + * (natural-language parsed, `connected` anchors resolved to canonical ids, + * an empty `where` dropped). + * @param hiddenIds - Ids this read must not return; apply BEFORE paging so + * `limit` stays exact. + * @param graphIndex - The active graph provider, for a `connected` plan. + * @returns The page's ids plus the stage that emptied it, or `null`. + */ + planFindPage?( + params: any, + hiddenIds: readonly string[], + graphIndex: unknown + ): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null> getIdsForTextQuery(query: string): Promise> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise diff --git a/tests/integration/find-planner-door.test.ts b/tests/integration/find-planner-door.test.ts new file mode 100644 index 00000000..964b13f9 --- /dev/null +++ b/tests/integration/find-planner-door.test.ts @@ -0,0 +1,137 @@ +/** + * @module tests/integration/find-planner-door + * @description The optional `MetadataIndexProvider.planFindPage` door. + * + * The stage doors each serve one stage, so a `find()` that consults three of + * them crosses into the index three times and marshals a result set at every + * crossing — a filter matching a hundred thousand rows builds a hundred + * thousand id strings to return a page of twenty-five. An index that can decide + * the stage order itself answers the page in one call. + * + * These pins hold the three properties that make such a door safe to add: + * + * 1. **Absent, nothing changes.** The reference index has no planner, and every + * find is served by the stage doors exactly as before. That is also what + * makes this engine the ordering oracle for any index that implements one. + * 2. **Present, it is asked first and its answer is used** — above the branch + * selection, with the params already normalized, the hidden ids passed, and + * the graph provider handed over. + * 3. **`null` is routing, not an answer.** A door that declines a shape leaves + * it to the path that always served it, and the result is unchanged. + * + * Plus the serving law: an empty page stamped `emptyAt: 'graph'` is re-verified + * against the adjacency before it is believed, so a not-serving graph refuses + * loudly instead of answering `[]` as truth. + */ +import { describe, it, expect, beforeAll, vi } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { generateTestVector } from '../helpers/test-factory' + +describe('find(): the optional planner door', () => { + let brain: Brainy + const anchor = 'planner-anchor' + let neighbourId = '' + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + await brain.add({ + id: anchor, + data: 'anchor', + type: NounType.Person, + metadata: { kind: 'anchor' }, + vector: generateTestVector() + }) + for (let i = 0; i < 12; i++) { + const id = await brain.add({ + id: `row-${i}`, + data: `row ${i}`, + type: NounType.Person, + metadata: { kind: 'note', rank: i }, + vector: generateTestVector() + }) + if (i === 0) neighbourId = id + await brain.relate({ from: anchor, to: id, type: VerbType.Knows }) + } + }) + + /** Install a planner door for one call, then remove it. */ + const withDoor = async ( + door: (...a: any[]) => Promise, + body: () => Promise + ): Promise => { + const index = (brain as any).metadataIndex + index.planFindPage = door + try { + return await body() + } finally { + delete index.planFindPage + } + } + + it('is absent on the reference index — every find is served by the stage doors', async () => { + expect((brain as any).metadataIndex.planFindPage).toBeUndefined() + const results = await brain.find({ where: { kind: 'note' }, limit: 5 }) + expect(results).toHaveLength(5) + }) + + it('is asked before the branches, with normalized params and the graph provider', async () => { + const door = vi.fn(async () => null) + await withDoor(door, async () => { + await brain.find({ where: { kind: 'note' }, limit: 5 }) + }) + expect(door).toHaveBeenCalledTimes(1) + const [params, hidden, graph] = door.mock.calls[0] as any[] + expect(params.where).toEqual({ kind: 'note' }) + expect(Array.isArray(hidden)).toBe(true) + expect(graph).toBe((brain as any).graphIndex) + }) + + it('uses the page it answers, hydrated and in the door\'s order', async () => { + const results = await withDoor( + async () => ({ ids: [neighbourId], emptyAt: 'none' as const }), + async () => brain.find({ where: { kind: 'note' }, limit: 5 }) + ) + expect(results).toHaveLength(1) + expect(results[0].entity.id).toBe(neighbourId) + }) + + it('a declining door changes nothing — the shape is served as it always was', async () => { + const withoutDoor = await brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 }) + const declined = await withDoor( + async () => null, + async () => brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 }) + ) + expect(declined.map((r) => r.entity.id)).toEqual(withoutDoor.map((r) => r.entity.id)) + }) + + it('re-verifies the adjacency before believing an empty graph answer', async () => { + const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive') + try { + const results = await withDoor( + async () => ({ ids: [], emptyAt: 'graph' as const }), + async () => brain.find({ connected: { from: anchor }, where: { kind: 'note' }, limit: 5 }) + ) + expect(results).toEqual([]) + expect(verify).toHaveBeenCalled() + } finally { + verify.mockRestore() + } + }) + + it('does not re-verify the adjacency for an empty the FILTER produced', async () => { + const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive') + verify.mockClear() + try { + const results = await withDoor( + async () => ({ ids: [], emptyAt: 'filter' as const }), + async () => brain.find({ where: { kind: 'note' }, limit: 5 }) + ) + expect(results).toEqual([]) + expect(verify).not.toHaveBeenCalled() + } finally { + verify.mockRestore() + } + }) +}) From f097cbf6f29cad03a34ca0c31c64fd86b53c9fdb Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 13:49:26 -0700 Subject: [PATCH 084/150] =?UTF-8?q?docs(releases):=20the=2010.4.7=20note?= =?UTF-8?q?=20=E2=80=94=20count=20ledgers=20can=20no=20longer=20race=20the?= =?UTF-8?q?mselves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/open-brainy.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index 21014f5b..e1dc83ce 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,17 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.7", + "date": "2026-09-01", + "headline": "Count ledgers can no longer race themselves", + "items": [ + "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", + "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", + "thumb": null + }, { "version": "10.4.6", "date": "2026-08-31", From 7ab670b525d62c86d7a39f81acdb1384fe2928ba Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 13:55:27 -0700 Subject: [PATCH 085/150] =?UTF-8?q?docs(releases):=20the=2011.0.4=20note?= =?UTF-8?q?=20=E2=80=94=20millisecond=20closes,=20storm-free=20rebuilds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/brainy.json b/releases/brainy.json index 17217a6e..c6c664a9 100644 --- a/releases/brainy.json +++ b/releases/brainy.json @@ -1,6 +1,18 @@ { "product": "brainy", "entries": [ + { + "version": "11.0.4", + "date": "2026-09-01", + "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", + "items": [ + "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", + "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", + "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." + ], + "url": null, + "thumb": null + }, { "version": "11.0.3", "date": "2026-09-01", From 8a2ebacf02ab62c4228a59c52e2b4201120f8f29 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 1 Sep 2026 16:03:33 -0700 Subject: [PATCH 086/150] =?UTF-8?q?fix(open):=20pending-embed=20recovery?= =?UTF-8?q?=20keeps=20the=20crash-recovery=20contract=20=E2=80=94=20foregr?= =?UTF-8?q?ound,=20bounded=20by=20the=20mark?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delta gate caught the backgrounded fold breaking six pinned crash-recovery cases: a reopened brain must have its markers re-armed when open() returns, and a background latch races every consumer of that contract. The backgrounding is reverted; the low-water mark stays — it is the part that kills the whole-history scan, and with it the foreground fold costs the log's tail on any brain that has ever drained. The unmarked first open after upgrade pays one full scan, once, and the open narrates it as its own step. --- src/brainy.ts | 72 ++++++++----------- .../pending-embed-low-water.test.ts | 16 ++--- 2 files changed, 37 insertions(+), 51 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index c9f24873..a49495f9 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1820,31 +1820,36 @@ export class Brainy implements BrainyInterface { // a deferred write's ack and its background embed DELAYED a vector; // this is where it lands. if (!this.isReadOnly) { - // BEHIND THE DOORS (the open pays nothing here): the bridge + the - // recovery fold run as one latched background task; the embed worker - // starts when it settles. A pending embed's outcome was always - // eventual — moving its recovery off the open's foreground changes - // when the worker starts, never whether a marker is honored. - // awaitPendingEmbeds() and close() wait on the latch first. - this._pendingEmbedRecovery = (async () => { - try { - await this.bridgeLegacyPendingEmbedSidecars() - await this.recoverPendingEmbedsFromLog() - if (this._pendingEmbedIds.size > 0) { - prodLog.info( - `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + - `session — resuming in the background` - ) - const t = setTimeout(() => this.kickEmbedWorker(), 0) - ;(t as { unref?: () => void }).unref?.() - } - } catch (err) { - prodLog.warn( - `[Brainy] pending-embed recovery failed: ${(err as Error).message} — ` + - `the log's markers remain durable; recovery retries next open` + // Foreground, as the crash-recovery contract pins it: a reopened brain + // has its markers re-armed when open() returns. The low-water mark + // bounds this to the log's tail on any brain that has ever drained — + // milliseconds — so the foreground cost is the unmarked first open + // only, once per upgraded brain. + try { + await step( + 'bridge-pending-embed-sidecars', + 'migrating any pre-log deferred-embed marker files into the generation log', + () => this.bridgeLegacyPendingEmbedSidecars() + ) + await step( + 'recover-pending-embeds', + 'folding the generation log\'s deferred-embed markers (from the low-water mark) into the pending set', + () => this.recoverPendingEmbedsFromLog() + ) + if (this._pendingEmbedIds.size > 0) { + prodLog.info( + `[Brainy] ${this._pendingEmbedIds.size} deferred embed(s) pending from a previous ` + + `session — resuming in the background` ) + const t = setTimeout(() => this.kickEmbedWorker(), 0) + ;(t as { unref?: () => void }).unref?.() } - })() + } catch (err) { + prodLog.warn( + `[Brainy] pending-embed recovery failed: ${(err as Error).message} — ` + + `the log's markers remain durable; recovery retries next open` + ) + } } // PHASE 4 of 5 — "VFS bootstrap": shutdown-hook registration, blob @@ -2418,8 +2423,6 @@ export class Brainy implements BrainyInterface { */ private static readonly PENDING_EMBED_LOWWATER_PATH = '_system/pending_embeds_lowwater.json' - /** Resolves when the background pending-embed recovery fold has settled (open arms it). */ - private _pendingEmbedRecovery: Promise | null = null /** * @description Mark a deferred embed pending (MT5): the id joins the @@ -2497,9 +2500,9 @@ export class Brainy implements BrainyInterface { * pending set last drained to empty — so a settled brain reads only the * facts since then, not its whole history. Without a mark (first open * after upgrade) it scans from generation 1, once; a stale-low mark costs - * a longer scan, never a marker. The fold runs BEHIND the doors (open - * arms it as a background task and the embed worker starts when it - * settles); {@link awaitPendingEmbeds} and close() wait for it first. + * a longer scan, never a marker. The fold stays on the open's foreground — + * the crash-recovery contract pins that a reopened brain has its markers + * re-armed when open() returns — and the mark is what makes that cheap. * It is SKIPPED WHOLESALE when the log has never had a v2 tail * ({@link FactLog.hasV2History} — v1 facts cannot carry marker records), * so pre-cutover brains pay nothing; on a mixed log the scan still reads @@ -2710,7 +2713,6 @@ export class Brainy implements BrainyInterface { * before I proceed" callers use this; nothing else ever needs to wait. */ public async awaitPendingEmbeds(): Promise { - if (this._pendingEmbedRecovery) await this._pendingEmbedRecovery while (this._pendingEmbedIds.size > 0 || this._embedWorkerFlight) { this.kickEmbedWorker() await (this._embedWorkerFlight ?? Promise.resolve()) @@ -19507,18 +19509,6 @@ export class Brainy implements BrainyInterface { * terminal releases have run. */ async close(): Promise { - if (this._pendingEmbedRecovery) { - // Settle the background marker fold before the durable steps — its scan - // is bounded by the low-water mark (a full scan happens at most once, - // on the first open after upgrade). - const settleStart = Date.now() - await this._pendingEmbedRecovery - const settleMs = Date.now() - settleStart - if (settleMs >= 1000) { - prodLog.info(`[Brainy] close: pending-embed recovery settled in ${settleMs}ms`) - } - this._pendingEmbedRecovery = null - } if (this._pendingEmbedIds.size === 0) await this.writeEmbedLowWater() let closeFailure: unknown = null try { diff --git a/tests/integration/pending-embed-low-water.test.ts b/tests/integration/pending-embed-low-water.test.ts index ff01b349..f966d0a1 100644 --- a/tests/integration/pending-embed-low-water.test.ts +++ b/tests/integration/pending-embed-low-water.test.ts @@ -6,9 +6,8 @@ * on the open's foreground — O(whole history) per open on long-lived brains. * Now: an advisory low-water mark (`_system/pending_embeds_lowwater.json`) * records the committed generation whenever the pending set drains to empty, - * recovery scans from `mark + 1`, and the fold runs behind the doors as a - * latched background task the worker, `awaitPendingEmbeds()` and `close()` - * wait on. The mark is advisory: stale-low costs a longer scan, never a + * recovery scans from `mark + 1` on the open's foreground — the crash-recovery + * contract keeps markers re-armed when open() returns. The mark is advisory: stale-low costs a longer scan, never a * marker — a pending embed enqueued before a crash is still recovered. */ import { describe, it, expect, afterEach, vi } from 'vitest' @@ -20,7 +19,7 @@ import { NounType } from '../../src/types/graphTypes' const LOWWATER_PATH = '_system/pending_embeds_lowwater.json' -describe('pending-embed recovery: bounded by the low-water mark, behind the doors', () => { +describe('pending-embed recovery: bounded by the low-water mark', () => { const roots: string[] = [] const dir = (): string => { const d = mkdtempSync(join(tmpdir(), 'brainy-lowwater-')) @@ -100,7 +99,6 @@ describe('pending-embed recovery: bounded by the low-water mark, behind the door await (brain as any).storage.releaseWriterLock() const brain2 = await open(root) - await (brain2 as any)._pendingEmbedRecovery expect(brain2.pendingEmbedCount()).toBeGreaterThan(0) await brain2.awaitPendingEmbeds() expect(brain2.pendingEmbedCount()).toBe(0) @@ -110,7 +108,7 @@ describe('pending-embed recovery: bounded by the low-water mark, behind the door await brain.close().catch(() => undefined) }) - it('open arms the fold as a background latch; awaitPendingEmbeds waits on it', async () => { + it('a reopened brain has its pending set settled when open() returns', async () => { const root = dir() const brain = await open(root) await brain.add({ id: 'a-row', data: 'some data', type: NounType.Thing }) @@ -118,10 +116,8 @@ describe('pending-embed recovery: bounded by the low-water mark, behind the door await brain.close() const brain2 = await open(root) - // The latch exists the moment init() returns (writable filesystem brain)… - expect((brain2 as any)._pendingEmbedRecovery).not.toBeNull() - // …and the barrier settles it before answering. - await brain2.awaitPendingEmbeds() + // The crash-recovery contract: markers are re-armed by open itself — + // no latch, no background race. (Here the drain landed, so zero.) expect(brain2.pendingEmbedCount()).toBe(0) await brain2.close() }) From eec90bdd698318aa7f47fbdce1e1c03ebec96b40 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 08:20:58 -0700 Subject: [PATCH 087/150] chore(release): 10.4.9 --- CHANGELOG.md | 11 +++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7154d5a2..16fb5786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.9](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.6...v10.4.9) (2026-09-02) + +- Merge branch 'fix/pending-embed-low-water' into rel/10.4.9-candidate (2648f56d) +- fix(open): pending-embed recovery keeps the crash-recovery contract — foreground, bounded by the mark (8a2ebacf) +- Merge branches 'fix/connected-find-order', 'fix/pending-embed-low-water' and 'fix/related-verb-array' into rel/10.4.9-candidate (d5147ed6) +- fix(graph): the verb fast paths honour every requested type, source, and target (6a89adc4) +- perf(open): pending-embed recovery is bounded by a low-water mark and runs behind the doors (88e79729) +- fix(find): connected finds are graph-first — neighbours, then the filter over those ids, then the page (077cbc0b) +- fix(storage): counts persistence is single-flight, coalesced, and never races its own temp file (5e3b343a) + + ### [10.4.6](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.5...v10.4.6) (2026-08-31) - fix(transact): metadata-index ops take their JSON-safe view at the crossing, not at construction (73500e7d) diff --git a/package-lock.json b/package-lock.json index 9e573da3..fc530baa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.6", + "version": "10.4.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.6", + "version": "10.4.9", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 51322998..f07bb94c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.6", + "version": "10.4.9", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From 297a3d76575771f60518a813b2c023e68bc9d707 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 08:30:01 -0700 Subject: [PATCH 088/150] =?UTF-8?q?docs(releases):=20the=2010.4.9=20note?= =?UTF-8?q?=20=E2=80=94=20graph-first=20finds,=20honest=20verb=20arrays,?= =?UTF-8?q?=20bounded=20recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/open-brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index e1dc83ce..582f4847 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,18 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.9", + "date": "2026-09-02", + "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", + "items": [ + "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", + "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", + "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", + "thumb": null + }, { "version": "10.4.7", "date": "2026-09-01", From 4f1e27c9a089f5dc5d3b20f7ba9fc52384ee0e28 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 08:32:21 -0700 Subject: [PATCH 089/150] =?UTF-8?q?docs(releases):=20the=2011.0.5=20note?= =?UTF-8?q?=20=E2=80=94=20graph-first=20finds=20in=20production,=20bounded?= =?UTF-8?q?=20recovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- releases/brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/brainy.json b/releases/brainy.json index c6c664a9..8f61c7f2 100644 --- a/releases/brainy.json +++ b/releases/brainy.json @@ -1,6 +1,18 @@ { "product": "brainy", "entries": [ + { + "version": "11.0.5", + "date": "2026-09-02", + "headline": "Graph-first finds in production, and opens that stop rescanning history", + "items": [ + "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", + "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", + "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." + ], + "url": null, + "thumb": null + }, { "version": "11.0.4", "date": "2026-09-01", From a8c5fbf9dc36a4ca12a5367aff17e9b6e1820305 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 08:43:39 -0700 Subject: [PATCH 090/150] fix(find): near() searches around the anchor's own vector, and refuses by name without one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proximity search fetched its anchor through get(), which omits vectors by default, then handed a zero-length vector to the index — every find({ near }) refused with a dimension mismatch, for every caller. Found by the Rust planner's first-contact pins comparing outcomes with and without the planner on a refused shape. The anchor is now fetched with its vector, and an anchor that has none refuses by name — a proximity search around an unvectored row has no meaning and must not fail inside the index. Pinned in tests/integration/find-near.test.ts. --- src/brainy.ts | 12 +++++++- tests/integration/find-near.test.ts | 48 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 tests/integration/find-near.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index c980661b..cc5413e0 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -15892,8 +15892,18 @@ export class Brainy implements BrainyInterface { ) } - const nearEntity = await this.get(params.near.id) + // The anchor's VECTOR is the query; get() omits vectors by default, which + // fed a zero-length vector to the index and refused every near() with a + // dimension mismatch. Ask for it, and refuse by name when the anchor has + // none — a proximity search around an unvectored row has no meaning. + const nearEntity = await this.get(params.near.id, { includeVectors: true }) if (!nearEntity) return [] + if (!nearEntity.vector || nearEntity.vector.length === 0) { + throw new Error( + `find({ near }): entity '${params.near.id}' has no vector to search around — ` + + `it was never embedded (or was unvectored). Embed it, or search with a query instead.` + ) + } const nearResults: [string, number][] = await this.index.search(nearEntity.vector, params.limit || 10) diff --git a/tests/integration/find-near.test.ts b/tests/integration/find-near.test.ts new file mode 100644 index 00000000..3fb235c8 --- /dev/null +++ b/tests/integration/find-near.test.ts @@ -0,0 +1,48 @@ +/** + * @module tests/integration/find-near + * @description find({ near }) searches around the anchor's OWN vector (10.4.10). + * + * The proximity search fetched its anchor without vectors and fed a + * zero-length vector to the index — every near() refused with a dimension + * mismatch, for every caller. Found by the Rust planner's first-contact pins + * (the planner declines `near`; the pin compared outcomes with and without + * it). Now the anchor is fetched with its vector, and an anchor without one + * refuses by name instead of failing inside the index. + */ +import { describe, it, expect, beforeAll } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType } from '../../src/types/graphTypes' +import { v5 } from '../../src/universal/uuid' +import { generateTestVector } from '../helpers/test-factory' + +describe('find({ near }) uses the anchor vector', () => { + let brain: Brainy + const anchorVector = generateTestVector() + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + await brain.add({ id: 'anchor', data: 'anchor row', type: NounType.Thing, vector: anchorVector }) + // A twin with the identical vector and a far row. + await brain.add({ id: 'twin', data: 'twin row', type: NounType.Thing, vector: [...anchorVector] }) + await brain.add({ id: 'far', data: 'far row', type: NounType.Thing, vector: generateTestVector() }) + }) + + it('returns the anchor\'s neighbours by its own vector', async () => { + const results = await brain.find({ near: { id: 'anchor' }, limit: 3 }) + expect(results.length).toBeGreaterThan(0) + const ids = results.map((r) => r.entity.id) + expect(ids).toContain(v5('twin')) + }) + + it('refuses by name when the anchor has no vector', async () => { + await brain.add({ + id: 'unvectored', + data: 'no vector here', + type: NounType.Thing, + deferEmbedding: true + }) + ;(brain as any).kickEmbedWorker = () => {} + await expect(brain.find({ near: { id: 'unvectored' }, limit: 3 })).rejects.toThrow(/has no vector to search around/) + }) +}) From f763317af7b191566381c7d2bbed825d02134770 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 09:09:30 -0700 Subject: [PATCH 091/150] =?UTF-8?q?feat(engine):=20a=20protected=20factory?= =?UTF-8?q?=20for=20the=20generation=20store=20=E2=80=94=20a=20subclass=20?= =?UTF-8?q?may=20substitute=20one=20that=20keeps=20the=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/brainy.ts | 13 ++- .../generation-store-factory.test.ts | 101 ++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 tests/integration/generation-store-factory.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index cc5413e0..4d7596d3 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1040,6 +1040,17 @@ export class Brainy implements BrainyInterface { } } + /** + * Factory hook for the generation store, so an engine built on top of this + * reference implementation can substitute a `GenerationStore` that keeps + * the same behavioural contract (for example, one backed by a native + * implementation) — overriding it never changes this engine's own + * behaviour, since the default implementation is unchanged. + */ + protected createGenerationStore(storage: BaseStorage): GenerationStore { + return new GenerationStore(storage) + } + /** * Initialize Brainy. * @@ -1297,7 +1308,7 @@ export class Brainy implements BrainyInterface { // guarantees indexes never observe rolled-back state. Reader-mode // instances skip recovery (readers never write; the next writer // repairs). - this.generationStore = new GenerationStore(this.storage) + this.generationStore = this.createGenerationStore(this.storage) const generationOpenResult = await step( 'generation-store.open', 'reading the generation manifest and committed ranges, opening the fact log and the ' + diff --git a/tests/integration/generation-store-factory.test.ts b/tests/integration/generation-store-factory.test.ts new file mode 100644 index 00000000..08b62619 --- /dev/null +++ b/tests/integration/generation-store-factory.test.ts @@ -0,0 +1,101 @@ +/** + * @module tests/integration/generation-store-factory + * @description Pins the `createGenerationStore` protected factory hook on + * `Brainy` ({@link Brainy.createGenerationStore}). The hook exists so an + * engine built on top of this reference implementation can substitute a + * `GenerationStore` that keeps the same behavioural contract; this suite + * proves two things: + * + * 1. A subclass overriding the hook is the ONLY path that constructs the + * generation store — it is called exactly once, with the same storage + * instance `performInit` holds — and the store the brain actually uses + * is the one the override returned. + * 2. The default (non-overridden) path is unaffected — proven here by + * confirming the base class still produces a plain `GenerationStore` + * wired to `brain.storage`, and separately by running the existing + * `db-mvcc` and `brainy-core.integration` suites unmodified against this + * change (they exercise generation-store behaviour end to end). + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { GenerationStore } from '../../src/db/generationStore.js' +import type { BaseStorage } from '../../src/storage/baseStorage.js' + +/** Typed access to the brain's private storage + generation-store fields (test injection point). */ +function internalsOf(brain: Brainy): { storage: BaseStorage; generationStore: GenerationStore } { + return brain as unknown as { storage: BaseStorage; generationStore: GenerationStore } +} + +/** + * A `GenerationStore` subclass that counts its own construction and + * remembers the storage instance it was built with, so the test can prove + * the hook is the sole construction path without mocking the module. + */ +class SpyGenerationStore extends GenerationStore { + static constructCount = 0 + static lastStorage: BaseStorage | undefined + + constructor(storage: BaseStorage) { + super(storage) + SpyGenerationStore.constructCount++ + SpyGenerationStore.lastStorage = storage + } +} + +/** A Brainy subclass overriding the factory hook — stands in for an engine built on the reference. */ +class BrainyWithSpyStore extends Brainy { + hookCallCount = 0 + hookStorageArg: BaseStorage | undefined + + protected override createGenerationStore(storage: BaseStorage): GenerationStore { + this.hookCallCount++ + this.hookStorageArg = storage + return new SpyGenerationStore(storage) + } +} + +describe('Brainy.createGenerationStore — protected factory hook', () => { + const brains: Brainy[] = [] + + afterEach(async () => { + SpyGenerationStore.constructCount = 0 + SpyGenerationStore.lastStorage = undefined + for (const brain of brains.splice(0)) { + try { + await brain.close() + } catch { + // already closed by the test + } + } + }) + + it('a subclass override is the sole construction path: called once, same storage instance, its store is the one the brain uses', async () => { + const brain = new BrainyWithSpyStore({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + brains.push(brain) + + // Called exactly once, through the hook. + expect(brain.hookCallCount).toBe(1) + expect(SpyGenerationStore.constructCount).toBe(1) + + // Same storage instance the base class holds — not a copy, not a different adapter. + const { storage, generationStore } = internalsOf(brain) + expect(brain.hookStorageArg).toBe(storage) + expect(SpyGenerationStore.lastStorage).toBe(storage) + + // The store the brain actually uses is the one the override returned. + expect(generationStore).toBeInstanceOf(SpyGenerationStore) + }) + + it('the default (non-overridden) path still produces a plain GenerationStore wired to the same storage', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + brains.push(brain) + + const { storage, generationStore } = internalsOf(brain) + expect(generationStore).toBeInstanceOf(GenerationStore) + // The default implementation constructs from the same storage the brain holds. + expect((generationStore as unknown as { storage: BaseStorage }).storage).toBe(storage) + }) +}) From 2633e8d5e1c59779985fa4b0f146e91deb826bc5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 09:10:44 -0700 Subject: [PATCH 092/150] docs(plugin): the planner door's hiddenIds contract is the answer, not the mechanism --- src/plugin.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/plugin.ts b/src/plugin.ts index fdad42f0..54d300bb 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -460,8 +460,10 @@ export interface MetadataIndexProvider { * @param params - The find params, already normalized by `find()` * (natural-language parsed, `connected` anchors resolved to canonical ids, * an empty `where` dropped). - * @param hiddenIds - Ids this read must not return; apply BEFORE paging so - * `limit` stays exact. + * @param hiddenIds - Ids this read must not return. The contract is the ANSWER, not the + * mechanism: a provider may subtract this set before paging, or derive the + * same exclusion from the params' visibility tiers itself — either way the + * page must equal the engine's own answer with none of these ids in it. * @param graphIndex - The active graph provider, for a `connected` plan. * @returns The page's ids plus the stage that emptied it, or `null`. */ From 9922631d1fd9052c7e332451bba5c4d69296308a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 09:09:33 -0700 Subject: [PATCH 093/150] ci: add the delta-gate workflow for the capped functional lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit workflow_dispatch, runs-on gate-functional — a host-mode, Bun-only lane with no Node.js runtime, so every step is plain git + bun in shell rather than a JS-based action. Clones candidate and control, runs the full vitest suite on each, enforces a >=3,000-collected guard per side, and diffs the two fail lists for genuinely new reds. The lane's own tripwire marker (host pressure — never our own red or green) is checked before the verdict is printed, and the job cleans up its own checkouts so repeat runs don't feed the lane's disk-budget trip. --- .forgejo/workflows/delta-gate.yml | 127 ++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 .forgejo/workflows/delta-gate.yml diff --git a/.forgejo/workflows/delta-gate.yml b/.forgejo/workflows/delta-gate.yml new file mode 100644 index 00000000..f9209e39 --- /dev/null +++ b/.forgejo/workflows/delta-gate.yml @@ -0,0 +1,127 @@ +name: Delta Gate + +# On-demand candidate-vs-control gate on the capped functional CI lane +# (label: gate-functional). That lane is Bun-only host-mode — there is no +# Node.js runtime available to it, so this workflow deliberately avoids every +# JS-based action (checkout/setup-node/setup-bun/upload-artifact all require +# one) and does everything with plain git + bun in shell steps instead. +# +# Verdict lines a caller should grep for in the run log: +# COLLECTED patch= control= — collection-truncation guard inputs +# NEW-RED-COUNT: — failures on candidate absent from control +# DELTA-GATE: CLEAN | NEW REDS | INVALID | STOPPED-BY-REGISTRY-TRIPWIRE +# +# The lane's own housekeeping stops the runner and drops a marker file when +# host pressure (I/O, registry latency, disk budget) trips — never ours to +# interpret as a red or a green. The final step checks for that marker before +# it says anything about pass/fail. + +on: + workflow_dispatch: + inputs: + candidate: + description: 'Candidate ref (branch or sha) to gate' + required: true + type: string + control: + description: 'Control sha to diff against' + required: true + type: string + +concurrency: + group: delta-gate + cancel-in-progress: false + +jobs: + delta-gate: + name: Delta gate — candidate vs control + runs-on: gate-functional + timeout-minutes: 120 + steps: + - name: Clean any residue from a prior run + run: rm -rf "ob-cand-${{ github.run_id }}" "ob-ctrl-${{ github.run_id }}" "/tmp/ob-${{ github.run_id }}-"* + + - name: Clone + test — candidate + id: patch + run: | + set -o pipefail + git clone --quiet "https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "ob-cand-${{ github.run_id }}" + cd "ob-cand-${{ github.run_id }}" + git checkout --quiet "${{ github.event.inputs.candidate }}" + git log --oneline -1 + bun install + rc=0 + bun x vitest run > "/tmp/ob-${{ github.run_id }}-patch.log" 2>&1 || rc=$? + echo "PATCH-RC:$rc" + grep -aE "Tests .*(passed|failed)" "/tmp/ob-${{ github.run_id }}-patch.log" | tail -1 + grep -aE "^ FAIL |^\s+×" "/tmp/ob-${{ github.run_id }}-patch.log" | sed -E "s/ [0-9]+ms$//" | sed -E "s/^\s+//" | sort -u > "/tmp/ob-${{ github.run_id }}-patch.fail" + echo "PATCH-FAILING:$(wc -l < "/tmp/ob-${{ github.run_id }}-patch.fail")" + + - name: Clone + test — control + id: control + run: | + set -o pipefail + git clone --quiet "https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "ob-ctrl-${{ github.run_id }}" + cd "ob-ctrl-${{ github.run_id }}" + git checkout --quiet "${{ github.event.inputs.control }}" + git log --oneline -1 + bun install + rc=0 + bun x vitest run > "/tmp/ob-${{ github.run_id }}-control.log" 2>&1 || rc=$? + echo "CONTROL-RC:$rc" + grep -aE "Tests .*(passed|failed)" "/tmp/ob-${{ github.run_id }}-control.log" | tail -1 + grep -aE "^ FAIL |^\s+×" "/tmp/ob-${{ github.run_id }}-control.log" | sed -E "s/ [0-9]+ms$//" | sed -E "s/^\s+//" | sort -u > "/tmp/ob-${{ github.run_id }}-control.fail" + echo "CONTROL-FAILING:$(wc -l < "/tmp/ob-${{ github.run_id }}-control.fail")" + + - name: Delta gate verdict + if: always() + run: | + set -o pipefail + + # The lane's own tripwire wins over anything we would otherwise say: + # a bare failure/timeout above with this marker present is host + # pressure, never a real red and never a real green. + if [ -f /srv/gate-lane/TRIPWIRE-STOPPED ]; then + echo "DELTA-GATE: STOPPED-BY-REGISTRY-TRIPWIRE" + head -1 /srv/gate-lane/TRIPWIRE-STOPPED + exit 3 + fi + + patch_log="/tmp/ob-${{ github.run_id }}-patch.log" + control_log="/tmp/ob-${{ github.run_id }}-control.log" + patch_fail="/tmp/ob-${{ github.run_id }}-patch.fail" + control_fail="/tmp/ob-${{ github.run_id }}-control.fail" + + if [ ! -s "$patch_log" ] || [ ! -s "$control_log" ]; then + echo "DELTA-GATE: INVALID — a leg produced no log (see the two steps above for the real cause)" + exit 2 + fi + + pt=$(grep -aoE "\(([0-9]+)\)$" "$patch_log" | tail -1 | tr -d "()") + ct=$(grep -aoE "\(([0-9]+)\)$" "$control_log" | tail -1 | tr -d "()") + echo "COLLECTED patch=${pt:-0} control=${ct:-0}" + if [ "${pt:-0}" -lt 3000 ] || [ "${ct:-0}" -lt 3000 ]; then + echo "DELTA-GATE: INVALID — truncated collection" + exit 2 + fi + + echo "=== NEW REDS ===" + comm -23 "$patch_fail" "$control_fail" + new=$(comm -23 "$patch_fail" "$control_fail" | wc -l) + echo "NEW-RED-COUNT:$new" + + echo "=== full candidate fail list ===" + cat "$patch_fail" + echo "=== full control fail list ===" + cat "$control_fail" + + if [ "$new" -eq 0 ]; then + echo "DELTA-GATE: CLEAN" + else + echo "DELTA-GATE: NEW REDS" + exit 1 + fi + + - name: Clean up (mind the lane's disk budget) + if: always() + run: rm -rf "ob-cand-${{ github.run_id }}" "ob-ctrl-${{ github.run_id }}" "/tmp/ob-${{ github.run_id }}-"* From 67ae0046de4063cf74bc9d57eb68adccd575ed5e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 09:36:12 -0700 Subject: [PATCH 094/150] ci(delta-gate): add a push fallback trigger alongside workflow_dispatch workflow_dispatch needs Actions-unit write on the dispatching credential; push does not, since Forgejo runs the workflow straight from the pushed ref's tree. A plain push to a rel/** or ci/** branch now also fires the gate, resolving candidate to the pushed commit and control to the last released, known-good tip (10.4.9) when the workflow_dispatch inputs aren't present. --- .forgejo/workflows/delta-gate.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/delta-gate.yml b/.forgejo/workflows/delta-gate.yml index f9209e39..c320594e 100644 --- a/.forgejo/workflows/delta-gate.yml +++ b/.forgejo/workflows/delta-gate.yml @@ -27,6 +27,12 @@ on: description: 'Control sha to diff against' required: true type: string + # workflow_dispatch needs Actions-unit write on the dispatching credential; + # push does not (it runs from the pushed ref's own tree), so a plain push + # to a release or CI branch is the fallback trigger while that grant is + # outstanding — see the ref-resolution step below for what it gates against. + push: + branches: ['rel/**', 'ci/**'] concurrency: group: delta-gate @@ -38,6 +44,21 @@ jobs: runs-on: gate-functional timeout-minutes: 120 steps: + - name: Resolve candidate/control refs + id: refs + run: | + candidate="${{ github.event.inputs.candidate }}" + control="${{ github.event.inputs.control }}" + # workflow_dispatch supplies both explicitly; a push event carries + # neither — fall back to the pushed commit as candidate and the + # last released, known-good tip (10.4.9) as control, so a plain + # push still produces a meaningful gate instead of an empty ref. + if [ -z "$candidate" ]; then candidate="${{ github.sha }}"; fi + if [ -z "$control" ]; then control="eec90bdd"; fi + echo "candidate=$candidate" >> "$GITHUB_OUTPUT" + echo "control=$control" >> "$GITHUB_OUTPUT" + echo "Resolved (trigger=${{ github.event_name }}): candidate=$candidate control=$control" + - name: Clean any residue from a prior run run: rm -rf "ob-cand-${{ github.run_id }}" "ob-ctrl-${{ github.run_id }}" "/tmp/ob-${{ github.run_id }}-"* @@ -47,7 +68,7 @@ jobs: set -o pipefail git clone --quiet "https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "ob-cand-${{ github.run_id }}" cd "ob-cand-${{ github.run_id }}" - git checkout --quiet "${{ github.event.inputs.candidate }}" + git checkout --quiet "${{ steps.refs.outputs.candidate }}" git log --oneline -1 bun install rc=0 @@ -63,7 +84,7 @@ jobs: set -o pipefail git clone --quiet "https://source.soulcraft.com/soulcraftlabs/open-brainy.git" "ob-ctrl-${{ github.run_id }}" cd "ob-ctrl-${{ github.run_id }}" - git checkout --quiet "${{ github.event.inputs.control }}" + git checkout --quiet "${{ steps.refs.outputs.control }}" git log --oneline -1 bun install rc=0 From b1c7054467139d225d85a2b658d92fbc391b630f Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 09:21:30 -0700 Subject: [PATCH 095/150] fix(find): the hybrid legs rank inside the filter, and only the page is read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hybrid find fuses a text leg and a semantic leg. The semantic leg already walked only the metadata filter's universe. The text leg did not: it ranked the WHOLE store, took the top `limit * 4`, read every one of those rows from canonical, and only then intersected with the filter. On a large store with a selective filter that is hundreds of rows read to return a handful — and a row matching both the query and the filter, but sitting outside the store-wide text prefix, was silently dropped. The same defect `find({ connected })` carried before the graph-first law, one leg over. Both legs now rank ids inside the universe and neither reads canonical. The text leg goes through a new optional `getIdsForTextQueryWithin` door on MetadataIndexProvider — the text twin of `filterIdsWithin`, so a native index can intersect its postings before any string crosses the boundary; the reference index implements it from its own posting-list merge, so the two doors can never disagree, and a provider without it is served by the whole-store answer intersected here. The fusion ranks shells, the page is cut from them, and canonical is read once for exactly that page — with the row rebuilt in full, so a hydrated row is indistinguishable from an eagerly-built one (same flattened fields, same entity, same match visibility, same key order). The eager forms of both legs stay for the search modes whose leg output IS the answer. Measured on the production recall shape (query + type list + `missing` negation + excludeVFS, limit 60) the old order read 241 rows in two batches to return one; the new order reads the page. Pinned in tests/integration/find-hybrid-filter-before-hydrate.test.ts. The oracle there is the pre-change pipeline itself, replayed on the same brain through the same doors: where the filter does not truncate the text leg the answer is identical — rows, order, scores, match visibility and row shape — across hybrid + where, + type list + excludeVFS + a `missing` negation, + connected, with and without offset. Where it does truncate, the correction is held by name: the old order's text leg contributed nothing at all, the new one returns the matching rows and paging reaches every one of them. The cost pins read the engine's own counters: one batchGet of `limit` ids, the whole-store text door never called, and what the text leg marshals bounded by the universe. --- src/brainy.ts | 353 +++++++---- src/plugin.ts | 22 + src/utils/metadataIndex.ts | 66 +- .../find-hybrid-filter-before-hydrate.test.ts | 580 ++++++++++++++++++ 4 files changed, 903 insertions(+), 118 deletions(-) create mode 100644 tests/integration/find-hybrid-filter-before-hydrate.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 4d7596d3..ad8bbf90 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -7651,6 +7651,13 @@ export class Brainy implements BrainyInterface { const searchMode = params.searchMode || 'auto' const limit = params.limit || 10 + // HYDRATE LAST (the hybrid path): its legs and its fusion rank IDS, and + // canonical is read at the two page exits below — never for a row the + // metadata filter is about to discard. This closure re-applies a hybrid + // row's match visibility once its entity is in hand; it is set only by + // the hybrid branch, so every other path hydrates unchanged. + let finishHybridRow: ((row: Result, pending: Result) => void) | undefined + // Handle text-only query (user explicitly wants text search) if (searchMode === 'text' && params.query && params.query.trim() !== '') { results = await this.executeTextSearch(params.query, limit * 2) @@ -7661,20 +7668,32 @@ export class Brainy implements BrainyInterface { } // Handle explicit hybrid or auto mode with query else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) { - // Zero-config hybrid: combine text + semantic search with RRF fusion - const [textResults, semanticResults] = await Promise.all([ - this.executeTextSearch(params.query, limit * 2), - this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) + // Zero-config hybrid: combine text + semantic search with RRF fusion. + // BOTH legs are held to the metadata filter's universe: the vector leg + // walks it as its candidate set, and the text leg ranks inside it + // instead of ranking the whole store and discarding what the filter + // would drop. Neither leg reads canonical — the page does, once. + const [textScored, semanticScored] = await Promise.all([ + this.executeTextSearchScored(params.query, limit * 2, preResolvedMetadataIds ?? undefined), + this.executeVectorSearchScored(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) ]) // Use user-specified alpha or auto-detect based on query length const alpha = params.hybridAlpha ?? this.autoAlpha(params.query) - // Tokenize query for match visibility + // Tokenize query for match visibility. The word list needs the entity, + // so it is computed on the page, at hydration. const queryWords = this.metadataIndex.tokenize(params.query) + const textResultIds = new Set(textScored.map((r) => r.id)) + finishHybridRow = (row, pending) => { + row.textMatches = this.findMatchingWords(row.entity, queryWords, textResultIds) + row.textScore = pending.textScore + row.semanticScore = pending.semanticScore + row.matchSource = pending.matchSource + } - // RRF fusion combines both result sets with match visibility - results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords) + // RRF fusion combines both ranked id sets with match visibility + results = this.rrfFusion(textScored, semanticScored, alpha) } // Handle direct vector search (no query text) - no hybrid needed else if (params.vector && !params.query) { @@ -7736,19 +7755,11 @@ export class Brainy implements BrainyInterface { const order = rankIndicesByScore(results.map(r => r.score), k, true) results = reorderByIndices(results, order).slice(offset, k) - // Batch-load entities only for the paginated results (10x faster on GCS) - const idsToLoad = results.filter(r => !r.entity).map(r => r.id) - if (idsToLoad.length > 0) { - const entitiesMap = await this.batchGet(idsToLoad) - for (const result of results) { - if (!result.entity) { - const entity = entitiesMap.get(result.id) - if (entity) { - result.entity = entity - } - } - } - } + // Batch-load entities only for the paginated results (10x faster on GCS). + // This is the hydrate-last seam for the deferring paths: a row that + // arrives as a ranked shell is rebuilt in full here — flattened + // fields, entity and match visibility — never `entity` alone. + results = await this.hydrateResultPage(results, finishHybridRow) // Early return if no other processing needed if (!params.connected && !params.fusion) { @@ -7854,8 +7865,13 @@ export class Brainy implements BrainyInterface { const finalOffset = params.offset || 0 - // Efficient pagination - only slice what we need (limit already defined above) - return results.slice(finalOffset, finalOffset + limit) + // Efficient pagination - only slice what we need (limit already defined + // above), THEN read canonical for the page. Rows that arrived hydrated + // pass straight through; a deferred path reads exactly these rows. + return await this.hydrateResultPage( + results.slice(finalOffset, finalOffset + limit), + finishHybridRow + ) })() // Index-integrity guard — applied ONCE here so every find() path (metadata, @@ -12949,6 +12965,31 @@ export class Brainy implements BrainyInterface { } } + /** + * The text-leg twin of {@link filterIdsWithinBelted}: rank `query` INSIDE the + * candidate universe, through the provider's own posting-list merge so the + * answer can never drift from `getIdsForTextQuery`'s. A provider without the + * door is served by its whole-store answer intersected here — the same rows + * in the same order, but it pays the whole-store marshal. + * + * @param query - The text query. + * @param ids - The candidate universe (the metadata filter's ids). + * @returns `{ id, matchCount }` rows inside `ids`, ranked by match count. + */ + private async textIdsWithinBelted( + query: string, + ids: readonly string[] + ): Promise> { + this.ensureIndexesLoaded(['metadata']) + const mip = this.metadataIndex as unknown as MetadataIndexProvider + if (typeof mip.getIdsForTextQueryWithin === 'function') { + return await mip.getIdsForTextQueryWithin(query, ids) + } + const within = new Set(ids) + const all = await this.metadataIndex.getIdsForTextQuery(query) + return all.filter((m) => within.has(m.id)) + } + async getIndexStatus(): Promise<{ initialized: boolean /** `true` once open()'s index-build-if-needed step has run. Named for API @@ -15841,6 +15882,44 @@ export class Brainy implements BrainyInterface { candidateIds?: string[], allowedIds?: OpaqueIdSet ): Promise[]> { + const scored = await this.executeVectorSearchScored(params, candidateIds, allowedIds) + + // Batch-load entities for 10-50x faster cloud storage performance + // GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster) + const entitiesMap = await this.batchGet(scored.map((s) => s.id)) + + const results: Result[] = [] + for (const { id, score } of scored) { + const entity = entitiesMap.get(id) + if (entity) { + results.push(this.createResult(id, score, entity)) + } + } + + return results + } + + /** + * The semantic leg WITHOUT hydration — ranked ids and their scores. + * + * The beam walk is already restricted to the candidate universe (that is what + * `candidateIds` / `allowedIds` are for), so the leg's cost is the walk. Its + * ROWS, though, are candidates for a fusion that will keep one page of them — + * so the hybrid path takes them unhydrated and reads exactly the page it + * returns. {@link executeVectorSearch} is the eager form, for the search modes + * whose leg output IS the answer. + * + * @param params - Find parameters (supplies the query/vector and the limit). + * @param candidateIds - Optional pre-resolved metadata universe (see + * {@link executeVectorSearch}). + * @param allowedIds - Optional opaque predicate-pushdown universe. + * @returns Ranked `{ id, score }` rows — no entity reads. + */ + private async executeVectorSearchScored( + params: FindParams, + candidateIds?: string[], + allowedIds?: OpaqueIdSet + ): Promise> { // Vector cold-read guard: before trusting a semantic/vector result, verify the // vector index actually SERVES a known persisted vector (one-shot per brain). // A pure semantic find({ query }) has no filter, so verifyMetadataLive never @@ -15866,21 +15945,10 @@ export class Brainy implements BrainyInterface { // HNSW search with optional metadata-first candidate filtering const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions) - // Batch-load entities for 10-50x faster cloud storage performance - // GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster) - const ids = searchResults.map(([id]) => id) - const entitiesMap = await this.batchGet(ids) - - const results: Result[] = [] - for (const [id, distance] of searchResults) { - const entity = entitiesMap.get(id) - if (entity) { - const score = Math.max(0, Math.min(1, 1 / (1 + distance))) - results.push(this.createResult(id, score, entity)) - } - } - - return results + return searchResults.map(([id, distance]) => ({ + id, + score: Math.max(0, Math.min(1, 1 / (1 + distance))) + })) } /** @@ -16180,30 +16248,64 @@ export class Brainy implements BrainyInterface { * @returns Array of Results with scores based on match count */ private async executeTextSearch(query: string, limit: number): Promise[]> { - const textMatches = await this.metadataIndex.getIdsForTextQuery(query) - if (textMatches.length === 0) return [] + const scored = await this.executeTextSearchScored(query, limit) + if (scored.length === 0) return [] - // Take top matches and load entities - const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering - const ids = topMatches.map(m => m.id) - const entitiesMap = await this.batchGet(ids) + // Batch-load entities for the whole leg — this is the eager form, kept for + // the text-only search mode whose results ARE the answer. + const entitiesMap = await this.batchGet(scored.map((s) => s.id)) - // Create results with scores based on match count - const maxMatches = topMatches[0]?.matchCount || 1 const results: Result[] = [] - - for (const match of topMatches) { - const entity = entitiesMap.get(match.id) + for (const { id, score } of scored) { + const entity = entitiesMap.get(id) if (entity) { - // Normalize score to 0-1 range based on match count - const score = match.matchCount / maxMatches - results.push(this.createResult(match.id, score, entity)) + results.push(this.createResult(id, score, entity)) } } return results } + /** + * The text leg WITHOUT hydration — ranked ids and their scores. + * + * FILTER BEFORE HYDRATE: when the caller already knows the candidate + * universe (the metadata filter's ids in a hybrid `find({ query, where })`), + * it is passed here and the word index ranks INSIDE that universe. The + * earlier order ranked the whole store, took the top `limit * 2`, hydrated + * every one of them, and only then intersected with the filter — so a + * filtered hybrid find on a large store hydrated hundreds of rows to return + * a handful, and a matching row outside the store-wide text prefix was + * silently dropped (the same defect `find({ connected })` had before the + * graph-first law). + * + * The score is the match count normalized against the top row's, so a + * restricted call normalizes against the top row IN THE UNIVERSE — the same + * rule applied to the set actually being ranked. + * + * @param query - Text query to search for. + * @param limit - Result budget; the leg keeps `limit * 2` for the fusion. + * @param candidateIds - Optional candidate universe to rank inside. + * @returns Ranked `{ id, score }` rows — no entity reads. + */ + private async executeTextSearchScored( + query: string, + limit: number, + candidateIds?: readonly string[] + ): Promise> { + const textMatches = candidateIds + ? await this.textIdsWithinBelted(query, candidateIds) + : await this.metadataIndex.getIdsForTextQuery(query) + if (textMatches.length === 0) return [] + + // Take top matches (more than the page, for the fusion to rank) + const topMatches = textMatches.slice(0, limit * 2) + + // Normalize score to 0-1 range based on match count + const maxMatches = topMatches[0]?.matchCount || 1 + return topMatches.map((m) => ({ id: m.id, score: m.matchCount / maxMatches })) + } + /** * Auto-detect optimal alpha for hybrid search * @@ -16230,55 +16332,56 @@ export class Brainy implements BrainyInterface { * * Formula: score(d) = sum(1 / (k + rank(d))) for each list * - * Now includes match visibility (textMatches, textScore, semanticScore, matchSource) + * Now includes match visibility (textScore, semanticScore, matchSource; the + * `textMatches` word list needs the entity and is filled at hydration). * - * @param textResults - Results from text search - * @param semanticResults - Results from semantic search + * HYDRATE LAST: both legs arrive as ranked ids + scores, and the fusion ranks + * ids — no entity is read here. The rows it returns are ranked SHELLS; the + * page is cut from them and only that page is read from canonical (see + * {@link hydrateResultPage}). The earlier order hydrated both legs in full — + * hundreds of rows — to return one page of them. + * + * @param textResults - Ranked ids + scores from text search + * @param semanticResults - Ranked ids + scores from semantic search * @param alpha - Weight for semantic (0=text only, 1=semantic only) - * @param queryWords - Original query words for match tracking * @param k - RRF constant (default: 60, standard in literature) - * @returns Fused results sorted by combined score with match visibility + * @returns Fused result shells sorted by combined score with match visibility */ - private async rrfFusion( - textResults: Result[], - semanticResults: Result[], + private rrfFusion( + textResults: ReadonlyArray<{ id: string; score: number }>, + semanticResults: ReadonlyArray<{ id: string; score: number }>, alpha: number, - queryWords: string[], k: number = 60 - ): Promise[]> { + ): Result[] { // Track scores and match details per entity interface MatchData { rrf: number textScore?: number semanticScore?: number - textMatches: string[] hasText: boolean hasSemantic: boolean } const matchData = new Map() - const entityMap = new Map>() // Text contribution (1 - alpha weight) const textWeight = 1 - alpha textResults.forEach((r, rank) => { const rrfScore = textWeight * (1 / (k + rank + 1)) - const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false } + const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false } existing.rrf += rrfScore existing.textScore = r.score // Original text search score (0-1) existing.hasText = true matchData.set(r.id, existing) - if (r.entity) entityMap.set(r.id, r.entity) }) // Semantic contribution (alpha weight) semanticResults.forEach((r, rank) => { const rrfScore = alpha * (1 / (k + rank + 1)) - const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false } + const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false } existing.rrf += rrfScore existing.semanticScore = r.score // Original semantic search score (0-1) existing.hasSemantic = true matchData.set(r.id, existing) - if (r.entity) entityMap.set(r.id, r.entity) }) // Sort by fused score @@ -16286,51 +16389,93 @@ export class Brainy implements BrainyInterface { .sort((a, b) => b[1].rrf - a[1].rrf) .map(([id, data]) => ({ id, data })) - // Build results - need to load any missing entities - const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id) - if (missingIds.length > 0) { - const loaded = await this.batchGet(missingIds) - for (const [id, entity] of loaded) { - entityMap.set(id, entity) - } - } - - // Performance: Build set of text result IDs for O(1) lookup - // This avoids re-extracting text for entities that weren't in text results - const textResultIds = new Set(textResults.map(r => r.id)) - - // Create final results with match visibility + // Create ranked shells with match visibility const results: Result[] = [] for (const { id, data } of sortedIds) { - const entity = entityMap.get(id) - if (entity) { - // Find which query words matched - uses fast path if entity wasn't in text results - const textMatches = this.findMatchingWords(entity, queryWords, textResultIds) - - // Determine match source - let matchSource: 'text' | 'semantic' | 'both' - if (data.hasText && data.hasSemantic) { - matchSource = 'both' - } else if (data.hasText) { - matchSource = 'text' - } else { - matchSource = 'semantic' - } - - // Create result with match visibility - const result = this.createResult(id, data.rrf, entity) - result.textMatches = textMatches - result.textScore = data.textScore - result.semanticScore = data.semanticScore - result.matchSource = matchSource - - results.push(result) + // Determine match source + let matchSource: 'text' | 'semantic' | 'both' + if (data.hasText && data.hasSemantic) { + matchSource = 'both' + } else if (data.hasText) { + matchSource = 'text' + } else { + matchSource = 'semantic' } + + const result = this.pendingResult(id, data.rrf) + result.textScore = data.textScore + result.semanticScore = data.semanticScore + result.matchSource = matchSource + + results.push(result) } return results } + /** + * A ranked candidate whose entity has NOT been read yet. + * + * The shell carries everything the ranking tail needs — the id, the score, + * and the match-visibility fields — and nothing that requires canonical. It + * is typed `Result` so it flows through the shared dedupe / visibility / + * filter / rank / page tail unchanged; {@link hydrateResultPage} turns the + * survivors into real results before any caller sees them, and find()'s + * index-integrity guard drops any row that never gained an entity. + * + * @param id - The candidate's canonical id. + * @param score - Its rank score. + */ + private pendingResult(id: string, score: number): Result { + return { id, score } as Result + } + + /** + * Read canonical for exactly the rows that need it — the hydrate-last seam. + * + * Rows that already carry an entity (the eager legs: metadata, text-only, + * semantic-only, proximity, graph) pass through untouched, so this is a no-op + * for every path that has not deferred. Rows that are shells are read in ONE + * batch and rebuilt through {@link createResult}, so a hydrated row is + * indistinguishable from an eagerly-built one — same flattened fields, same + * `entity`, same key order — with `finish` re-applying the fields only the + * deferring path knows about (a hybrid row's match visibility). + * + * A shell whose id has no canonical row is dropped, exactly as the eager legs + * dropped it; find()'s index-integrity guard makes the same judgement on the + * page it returns. + * + * @param rows - The page's rows, ranked and paged already. + * @param finish - Applied to each rebuilt row, with its shell, after the + * flattened fields are set. + * @returns The page with every surviving row hydrated. + */ + private async hydrateResultPage( + rows: Result[], + finish?: (row: Result, pending: Result) => void + ): Promise[]> { + const pendingIds: string[] = [] + for (const row of rows) { + if (!row.entity) pendingIds.push(row.id) + } + if (pendingIds.length === 0) return rows + + const entitiesMap = await this.batchGet(pendingIds) + const hydrated: Result[] = [] + for (const row of rows) { + if (row.entity) { + hydrated.push(row) + continue + } + const entity = entitiesMap.get(row.id) + if (!entity) continue + const filled = this.createResult(row.id, row.score, entity, row.explanation) + finish?.(filled, row) + hydrated.push(filled) + } + return hydrated + } + /** * Find which query words match in an entity's text content * diff --git a/src/plugin.ts b/src/plugin.ts index 54d300bb..64abfe26 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -473,6 +473,28 @@ export interface MetadataIndexProvider { graphIndex: unknown ): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null> getIdsForTextQuery(query: string): Promise> + /** + * @description OPTIONAL: score `query` over `ids` ONLY — the text-leg twin of + * {@link filterIdsWithin}, and the door a hybrid `find({ query, where })` + * walks. The metadata filter's universe is the candidate set there, so the + * text leg must cost O(|ids|) membership checks and marshal at most `|ids|` + * rows, never the whole posting list of every query word. A native index + * intersects its own postings with the candidate set (membership by entity + * int) before any string crosses the boundary; the reference index answers + * from its own `getIdsForTextQuery`, so the two doors can never disagree. + * Absent → Brainy intersects `getIdsForTextQuery`'s answer with `ids` itself + * (correct, and still hydrate-last, but it marshals the whole answer). + * + * The answer keeps `getIdsForTextQuery`'s contract: `{ id, matchCount }` + * sorted by `matchCount` descending, ties in the order the whole-store answer + * would have produced. Only rows in `ids` may appear. + * @param query - The same text query accepted by `getIdsForTextQuery`. + * @param ids - The candidate ids (canonical). The answer is a subset. + */ + getIdsForTextQueryWithin?( + query: string, + ids: readonly string[] + ): Promise> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise getFilterFields(): Promise diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 0fd312e2..a3aa7679 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1509,11 +1509,56 @@ export class MetadataIndexManager implements MetadataIndexProvider { * @returns Array of { id, matchCount } sorted by matchCount descending */ async getIdsForTextQuery(query: string): Promise> { + return this.scoreTextQuery(query) + } + + /** + * Score a text query over `ids` ONLY — the reference implementation of the + * optional `getIdsForTextQueryWithin` door (see + * {@link import('../plugin.js').MetadataIndexProvider}). The hybrid + * `find({ query, where })` path passes the metadata filter's universe here so + * the text leg ranks INSIDE that universe instead of ranking the whole store + * and discarding the rows the filter would have dropped. + * + * It answers from the same posting-list merge as {@link getIdsForTextQuery}, + * with the candidate membership applied as each word's postings are counted, + * so the two doors can never disagree: the answer is exactly the whole-store + * answer restricted to `ids`, in the same order. + * + * @param query - Text query to search for. + * @param ids - Candidate entity ids; only these may appear in the answer. + * @returns Array of { id, matchCount } sorted by matchCount descending. + */ + async getIdsForTextQueryWithin( + query: string, + ids: readonly string[] + ): Promise> { + if (ids.length === 0) return [] + return this.scoreTextQuery(query, new Set(ids)) + } + + /** + * The one posting-list merge behind both text doors. + * + * Each query word contributes AT MOST one match per entity (a posting list + * can name an id more than once), and entities are ranked by how many of the + * query's words they matched. `within`, when given, restricts the count to + * those candidates — applied during the merge, so a restricted call never + * materializes a whole-store match map. + * + * @param query - Text query to search for. + * @param within - Optional candidate universe; absent = the whole store. + * @returns Array of { id, matchCount } sorted by matchCount descending. + */ + private async scoreTextQuery( + query: string, + within?: ReadonlySet + ): Promise> { const queryWords = this.tokenize(query) if (queryWords.length === 0) return [] - // Get IDs for each word hash - const wordIdSets: Map[] = [] + // Count matches per entity, one word's postings at a time. + const matchCounts = new Map() for (const word of queryWords) { const wordHash = this.hashWord(word) let ids: string[] @@ -1529,19 +1574,12 @@ export class MetadataIndexManager implements MetadataIndexProvider { throw err } } - const idSet = new Map() + // One count per (word, entity) — dedupe this word's postings first. + const counted = new Set() for (const id of ids) { - idSet.set(id, 1) - } - wordIdSets.push(idSet) - } - - if (wordIdSets.length === 0) return [] - - // Count matches per entity - const matchCounts = new Map() - for (const idSet of wordIdSets) { - for (const [id] of idSet) { + if (counted.has(id)) continue + counted.add(id) + if (within && !within.has(id)) continue matchCounts.set(id, (matchCounts.get(id) || 0) + 1) } } diff --git a/tests/integration/find-hybrid-filter-before-hydrate.test.ts b/tests/integration/find-hybrid-filter-before-hydrate.test.ts new file mode 100644 index 00000000..252fe89b --- /dev/null +++ b/tests/integration/find-hybrid-filter-before-hydrate.test.ts @@ -0,0 +1,580 @@ +/** + * @module tests/integration/find-hybrid-filter-before-hydrate + * @description FILTER BEFORE HYDRATE, applied to the hybrid `find({ query })` path. + * + * A hybrid find fuses two legs. The semantic leg already walked only the + * metadata filter's universe (`candidateIds` / `allowedIds`). The TEXT leg did + * not: it ranked the WHOLE store, took the top `limit * 4`, read every one of + * those rows from canonical, and only then intersected with the filter — so a + * filtered hybrid find on a large store read hundreds of rows to return a + * handful of them, and a matching row outside the store-wide text prefix was + * silently dropped. That is the same defect `find({ connected })` carried + * before the graph-first law, one leg over. + * + * Both halves are pinned here. + * + * THE ANSWER. Where the filter did not truncate the text leg — the universe + * covers every text match, so both orders rank the same rows — the new + * pipeline's answer is IDENTICAL to the old one's: same rows, same order, same + * scores, same match visibility, same row shape. The oracle below is the + * pre-change pipeline itself, replayed on the same brain through the same + * doors, so the comparison is against what actually ran, not a remembered + * expectation. + * + * THE CORRECTION. Where the filter DID truncate it — the query's words are + * common outside the universe — the old order let the text leg contribute + * nothing at all: every row it ranked was discarded by the filter, and the + * answer came from the semantic leg alone. The new order ranks inside the + * universe, so the text leg contributes the rows it always should have. + * + * THE COST. Canonical is read for exactly the page: one batch, `limit` rows, + * never the legs. And the text leg is asked about the universe's ids only — + * what it marshals is bounded by the universe, not by the store. + */ +import { describe, it, expect, beforeAll, vi } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking' +import { resolveEntityId } from '../../src/utils/idNormalization' + +/** Embedding width of the default model — the row vectors must match it. */ +const DIM = 384 + +/** + * A deterministic, per-row-distinct unit vector. Distinct so the semantic leg + * has a real ranking to produce (identical vectors would make its order a tie + * break), deterministic so the oracle and the pipeline see the same one. + */ +function seededVector(seed: number): number[] { + const v = new Array(DIM) + for (let i = 0; i < DIM; i++) { + v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3 + } + const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0)) + return v.map((x) => x / magnitude) +} + +/** The fields a caller reads off a hybrid row — the whole comparable surface. */ +function project(rows: any[]): any[] { + return rows.map((r) => ({ + id: r.id, + score: r.score, + type: r.type, + metadata: r.metadata, + textMatches: r.textMatches, + textScore: r.textScore, + semanticScore: r.semanticScore, + matchSource: r.matchSource + })) +} + +/** + * The PRE-CHANGE hybrid pipeline, replayed on a live brain through the same + * provider doors it used: whole-store text ranking with both legs hydrated in + * full, RRF fusion, then the metadata intersection, then the page. + * + * Supports the shapes these pins exercise (query + where/type/excludeVFS + + * connected + offset); `orderBy`, `fusion` and `near` are not replayed. + */ +async function legacyHybridFind(brain: any, params: any): Promise { + const index = brain.metadataIndex + const limit = params.limit ?? 10 + const offset = params.offset ?? 0 + const hasFilter = Boolean( + params.where || params.type || params.subtype || params.service || params.excludeVFS + ) + + let preResolvedMetadataIds: string[] | null = null + let preResolvedFilter: any = null + let graphFirstIds: string[] | null = null + + if (params.connected) { + // find() normalizes the anchors to canonical ids before this stage runs. + const anchored = { + ...params, + connected: { + ...params.connected, + ...(params.connected.from && { from: resolveEntityId(params.connected.from) }), + ...(params.connected.to && { to: resolveEntityId(params.connected.to) }) + } + } + graphFirstIds = await brain.resolveConnectedIds(anchored) + if (graphFirstIds!.length > 0 && hasFilter) { + preResolvedFilter = brain.buildMetadataFilter(params) + graphFirstIds = await brain.filterIdsWithinBelted(preResolvedFilter, graphFirstIds) + } + if (graphFirstIds!.length === 0) return [] + preResolvedMetadataIds = graphFirstIds + } else if (hasFilter) { + preResolvedFilter = brain.buildMetadataFilter(params) + preResolvedMetadataIds = await brain.filterIdsBelted(preResolvedFilter) + if (preResolvedMetadataIds!.length === 0) return [] + } + + // Text leg — the whole store, then the top `limit * 4`, hydrated in full. + const allTextMatches = await index.getIdsForTextQuery(params.query) + const topMatches = allTextMatches.slice(0, limit * 2 * 2) + const maxMatches = topMatches[0]?.matchCount || 1 + const textEntities = await brain.batchGet(topMatches.map((m: any) => m.id)) + const textResults = topMatches + .filter((m: any) => textEntities.has(m.id)) + .map((m: any) => ({ id: m.id, score: m.matchCount / maxMatches })) + + // Semantic leg — the beam walk over the universe, hydrated in full. + const vector = await brain.embed(params.query) + const searchOptions = preResolvedMetadataIds ? { candidateIds: preResolvedMetadataIds } : undefined + const searchResults: [string, number][] = await brain.index.search( + vector, + limit * 2, + undefined, + searchOptions + ) + const semanticEntities = await brain.batchGet(searchResults.map(([id]) => id)) + const semanticResults = searchResults + .filter(([id]) => semanticEntities.has(id)) + .map(([id, distance]) => ({ id, score: Math.max(0, Math.min(1, 1 / (1 + distance))) })) + + // RRF fusion, with the match visibility the rows carried. + const alpha = params.hybridAlpha ?? brain.autoAlpha(params.query) + const k = 60 + const matchData = new Map() + const textWeight = 1 - alpha + textResults.forEach((r: any, rank: number) => { + const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false } + existing.rrf += textWeight * (1 / (k + rank + 1)) + existing.textScore = r.score + existing.hasText = true + matchData.set(r.id, existing) + }) + semanticResults.forEach((r: any, rank: number) => { + const existing = matchData.get(r.id) || { rrf: 0, hasText: false, hasSemantic: false } + existing.rrf += alpha * (1 / (k + rank + 1)) + existing.semanticScore = r.score + existing.hasSemantic = true + matchData.set(r.id, existing) + }) + + const queryWords: string[] = index.tokenize(params.query) + const textResultIds = new Set(textResults.map((r: any) => r.id)) + const fusedIds = Array.from(matchData.entries()) + .sort((a, b) => b[1].rrf - a[1].rrf) + .map(([id, data]) => ({ id, data })) + + const allEntities = await brain.batchGet(fusedIds.map((f) => f.id)) + let rows: any[] = [] + for (const { id, data } of fusedIds) { + const entity = allEntities.get(id) + if (!entity) continue + const textContent = textResultIds.has(id) + ? index.extractTextContent({ data: entity.data, metadata: entity.metadata }).toLowerCase() + : null + rows.push({ + id, + score: data.rrf, + type: entity.type, + metadata: entity.metadata, + textMatches: + textContent === null ? [] : queryWords.filter((w) => textContent.includes(w.toLowerCase())), + textScore: data.textScore, + semanticScore: data.semanticScore, + matchSource: data.hasText && data.hasSemantic ? 'both' : data.hasText ? 'text' : 'semantic' + }) + } + + // The metadata intersection — after the legs, as it was. + if (preResolvedMetadataIds && preResolvedFilter) { + const filteredIdSet = new Set(preResolvedMetadataIds) + rows = rows.filter((r) => filteredIdSet.has(r.id)) + } + if (graphFirstIds !== null) { + const neighbourSet = new Set(graphFirstIds) + rows = rows.filter((r) => neighbourSet.has(r.id)) + } + + // Rank to the page, then cut it. + const order = rankIndicesByScore( + rows.map((r) => r.score), + offset + limit, + true + ) + return reorderByIndices(rows, order).slice(offset, offset + limit) +} + +/** + * FIXTURE A — the filter's universe covers every text match, so the two orders + * rank exactly the same rows and the answers must be identical. + */ +describe('hybrid find: filter before hydrate — the answer is unchanged', () => { + let brain: Brainy + const QUERY = 'orbital telemetry' + const MATCHES = 24 + const FILLER = 120 + const OUTSIDE = 30 + const VFS = 10 + const RETRACTED = 6 + const anchor = 'array-anchor' + const matchIds: string[] = [] + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + + let seed = 1 + await brain.add({ + id: anchor, + data: 'ground station anchor record', + type: NounType.Thing, + metadata: { lane: 'alpha', role: 'anchor' }, + vector: seededVector(seed++) + }) + + // Rows the query's words actually match — all inside every filter below. + for (let i = 0; i < MATCHES; i++) { + const id = `match-${i}` + await brain.add({ + id, + data: `orbital telemetry packet ${i} recorded downlink`, + type: NounType.Document, + metadata: { lane: 'alpha', rank: i }, + vector: seededVector(seed++) + }) + matchIds.push(resolveEntityId(id)) + await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo }) + } + // Rows inside the universe that the query's words do NOT match. + for (let i = 0; i < FILLER; i++) { + await brain.add({ + id: `filler-${i}`, + data: `cistern ledger entry ${i} archived`, + type: NounType.Document, + metadata: { lane: 'alpha', rank: 1000 + i }, + vector: seededVector(seed++) + }) + } + // Rows outside the universe. + for (let i = 0; i < OUTSIDE; i++) { + await brain.add({ + id: `outside-${i}`, + data: `unrelated dossier ${i}`, + type: NounType.Person, + metadata: { lane: 'beta' }, + vector: seededVector(seed++) + }) + } + // VFS infrastructure rows — excluded by excludeVFS. + for (let i = 0; i < VFS; i++) { + await brain.add({ + id: `vfs-${i}`, + data: `mounted path ${i}`, + type: NounType.Document, + metadata: { lane: 'alpha', vfsType: 'file' }, + vector: seededVector(seed++) + }) + } + // Retracted rows — excluded by a `missing` negation. + for (let i = 0; i < RETRACTED; i++) { + await brain.add({ + id: `retracted-${i}`, + data: `withdrawn note ${i}`, + type: NounType.Document, + metadata: { lane: 'alpha', retracted: true }, + vector: seededVector(seed++) + }) + } + + // The reference index has no opaque-set door, so the pipeline and the + // oracle both restrict the beam walk with the materialized candidate ids. + expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function') + }) + + it('the fixture does not truncate the text leg — the universe covers every text match', async () => { + const index = (brain as any).metadataIndex + const textMatches = await index.getIdsForTextQuery(QUERY) + expect(textMatches).toHaveLength(MATCHES) + const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' }) + const inUniverse = new Set(universe) + for (const m of textMatches) expect(inUniverse.has(m.id)).toBe(true) + }) + + it('hybrid + where: identical rows, identical order, identical scores', async () => { + const params = { query: QUERY, where: { lane: 'alpha' }, limit: 8 } + const expected = await legacyHybridFind(brain as any, params) + const actual = await brain.find(params as any) + expect(actual.length).toBe(expected.length) + expect(project(actual)).toEqual(expected) + }) + + it('hybrid + where + offset: identical page two', async () => { + const params = { query: QUERY, where: { lane: 'alpha' }, limit: 6, offset: 6 } + const expected = await legacyHybridFind(brain as any, params) + const actual = await brain.find(params as any) + expect(actual.length).toBe(expected.length) + expect(project(actual)).toEqual(expected) + }) + + it('hybrid + type list + excludeVFS + a `missing` negation: identical', async () => { + const params = { + query: QUERY, + type: [NounType.Document, NounType.Person], + excludeVFS: true, + where: { lane: 'alpha', retracted: { missing: true } }, + limit: 8 + } + const expected = await legacyHybridFind(brain as any, params) + const actual = await brain.find(params as any) + expect(actual.length).toBe(expected.length) + expect(project(actual)).toEqual(expected) + for (const r of actual) { + expect(r.metadata.retracted).toBeUndefined() + expect(r.metadata.vfsType).toBeUndefined() + } + }) + + it('hybrid + type list + excludeVFS + a `missing` negation, offset: identical', async () => { + const params = { + query: QUERY, + type: [NounType.Document, NounType.Person], + excludeVFS: true, + where: { lane: 'alpha', retracted: { missing: true } }, + limit: 5, + offset: 5 + } + const expected = await legacyHybridFind(brain as any, params) + const actual = await brain.find(params as any) + expect(actual.length).toBe(expected.length) + expect(project(actual)).toEqual(expected) + }) + + it('hybrid + connected: identical, and never a non-neighbour', async () => { + const params = { + query: QUERY, + connected: { from: anchor, direction: 'out' as const }, + where: { lane: 'alpha' }, + limit: 8 + } + const expected = await legacyHybridFind(brain as any, params) + const actual = await brain.find(params as any) + expect(actual.length).toBe(expected.length) + expect(project(actual)).toEqual(expected) + const neighbours = new Set(matchIds) + for (const r of actual) expect(neighbours.has(r.id)).toBe(true) + }) + + it('a hydrated hybrid row is shaped exactly as an eagerly-built one', async () => { + const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 8 } as any) + const row = rows[0] + expect(Object.keys(row)).toEqual([ + 'id', + 'score', + 'type', + 'subtype', + 'visibility', + 'metadata', + 'data', + 'confidence', + 'weight', + '_rev', + 'entity', + 'textMatches', + 'textScore', + 'semanticScore', + 'matchSource' + ]) + // The flattened fields are projections of the entity, as always. + expect(row.entity).toBeDefined() + expect(row.type).toBe(row.entity.type) + expect(row.metadata).toBe(row.entity.metadata) + expect(row.data).toBe(row.entity.data) + expect(row._rev).toBe(row.entity._rev) + // The match visibility survives the deferral — every leg's fields, on the + // rows that leg contributed, exactly as the eager pipeline set them. + expect(['text', 'semantic', 'both']).toContain(row.matchSource) + for (const r of rows) { + if (r.matchSource === 'semantic') { + expect(r.textMatches).toEqual([]) + expect(r.textScore).toBeUndefined() + } else { + expect(r.textMatches).toEqual(['orbital', 'telemetry']) + expect(typeof r.textScore).toBe('number') + } + if (r.matchSource === 'text') { + expect(r.semanticScore).toBeUndefined() + } else { + expect(typeof r.semanticScore).toBe('number') + } + } + }) + + it('reads canonical for the page only — one batch, `limit` rows', async () => { + // Warm any first-read verification before the counters are read. + await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any) + + const hydrate = vi.spyOn(brain as any, 'batchGet') + try { + const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any) + expect(results).toHaveLength(10) + expect(hydrate).toHaveBeenCalledTimes(1) + expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10) + } finally { + hydrate.mockRestore() + } + }) + + it('asks the text index about the universe only, never the whole store', async () => { + const index = (brain as any).metadataIndex + const wholeStore = vi.spyOn(index, 'getIdsForTextQuery') + const within = vi.spyOn(index, 'getIdsForTextQueryWithin') + try { + await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any) + expect(wholeStore).not.toHaveBeenCalled() + expect(within).toHaveBeenCalledTimes(1) + + const askedIds = within.mock.calls[0][1] as string[] + const universe = await (brain as any).filterIdsBelted({ lane: 'alpha' }) + expect(askedIds).toHaveLength(universe.length) + + // What the text leg marshals is bounded by the universe, not the store. + const marshalled = (await within.mock.results[0].value) as unknown[] + expect(marshalled.length).toBeLessThanOrEqual(universe.length) + expect(marshalled).toHaveLength(MATCHES) + } finally { + wholeStore.mockRestore() + within.mockRestore() + } + }) + + it('the two text doors agree: within is the whole-store answer restricted', async () => { + const index = (brain as any).metadataIndex + const universe: string[] = await (brain as any).filterIdsBelted({ + lane: 'alpha', + retracted: { missing: true } + }) + const inUniverse = new Set(universe) + const whole = await index.getIdsForTextQuery(QUERY) + const within = await index.getIdsForTextQueryWithin(QUERY, universe) + expect(within).toEqual(whole.filter((m: any) => inUniverse.has(m.id))) + expect(await index.getIdsForTextQueryWithin(QUERY, [])).toEqual([]) + }) +}) + +/** + * FIXTURE B — the query's words are common OUTSIDE the universe, so the old + * order's text leg was entirely consumed by rows the filter then discarded. + * This is the corrected behaviour, held by name. + */ +describe('hybrid find: the text leg ranks inside the filter, not around it', () => { + let brain: Brainy + const QUERY = 'orbital telemetry drift' + const NOISE = 150 + const KEEP = 15 + const keepIds: string[] = [] + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + + let seed = 5000 + // Added FIRST and matching one more query word, so they lead the + // store-wide text ranking outright — and none of them pass the filter. + for (let i = 0; i < NOISE; i++) { + await brain.add({ + id: `noise-${i}`, + data: `orbital telemetry drift report ${i}`, + type: NounType.Document, + metadata: { lane: 'beta' }, + vector: seededVector(seed++) + }) + } + for (let i = 0; i < KEEP; i++) { + const id = `keep-${i}` + await brain.add({ + id, + data: `orbital telemetry summary ${i}`, + type: NounType.Document, + metadata: { lane: 'alpha' }, + vector: seededVector(seed++) + }) + keepIds.push(resolveEntityId(id)) + } + }) + + it('the old order let the filter consume the whole text leg', async () => { + const index = (brain as any).metadataIndex + const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) + expect(universe).toHaveLength(KEEP) + const inUniverse = new Set(universe) + + // The store-wide prefix the old text leg took (limit 10 → limit * 4). + const prefix = (await index.getIdsForTextQuery(QUERY)).slice(0, 40) + expect(prefix).toHaveLength(40) + expect(prefix.filter((m: any) => inUniverse.has(m.id))).toHaveLength(0) + + // Every row the old text leg ranked was then discarded by the filter, so + // the old answer carried NO text contribution at all — fifteen rows that + // match the query's words exactly, and not one of them reached the page + // through the text leg. What the old order returned was whatever the + // semantic leg alone happened to reach. + const legacy = await legacyHybridFind(brain as any, { + query: QUERY, + where: { lane: 'alpha' }, + limit: 10 + }) + for (const r of legacy) { + expect(r.matchSource).toBe('semantic') + expect(r.textScore).toBeUndefined() + expect(r.textMatches).toEqual([]) + } + }) + + it('the new order ranks the text leg inside the universe', async () => { + const results = await brain.find({ + query: QUERY, + where: { lane: 'alpha' }, + limit: 10 + } as any) + + expect(results).toHaveLength(10) + const keeps = new Set(keepIds) + for (const r of results) { + expect(keeps.has(r.id)).toBe(true) + expect(r.metadata.lane).toBe('alpha') + // The text leg is the contributor the old order threw away. + expect(['text', 'both']).toContain(r.matchSource) + expect(r.textScore).toBe(1) + expect(r.textMatches).toEqual(['orbital', 'telemetry']) + } + }) + + it('paging reaches every matching row the old order could not see', async () => { + const seen = new Set() + for (let offset = 0; offset < KEEP; offset += 5) { + const page = await brain.find({ + query: QUERY, + where: { lane: 'alpha' }, + limit: 5, + offset + } as any) + expect(page).toHaveLength(5) + for (const r of page) { + expect(seen.has(r.id)).toBe(false) + seen.add(r.id) + } + } + expect(seen.size).toBe(KEEP) + expect([...seen].sort()).toEqual([...keepIds].sort()) + }) + + it('reads canonical for the page only, on the truncating shape too', async () => { + await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 1 } as any) + + const hydrate = vi.spyOn(brain as any, 'batchGet') + try { + const results = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 10 } as any) + expect(results).toHaveLength(10) + expect(hydrate).toHaveBeenCalledTimes(1) + expect((hydrate.mock.calls[0][0] as string[]).length).toBe(10) + } finally { + hydrate.mockRestore() + } + }) +}) From 905c267c47a9515abcde4a713b3c575ec730e7b2 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 09:28:44 -0700 Subject: [PATCH 096/150] fix(find): a page the metadata block already cut is not cut again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `find({ query, connected, where, offset })` answered [] for every page but the first. The metadata block ranks the fused candidates and CUTS the page itself — rows [offset, offset+limit) — and then returns early. Two shapes do not take that early return, `connected` and `fusion`, and they fell through to the tail, which sliced the already-cut page by `offset` a second time: a five-row page sliced at offset five is nothing at all. Every page after the first was empty, and the caller had no way to tell that from "no more rows". The block now records that it consumed the offset, and the tail returns the page it was handed instead of re-cutting it. Nothing changes at offset 0, where the second slice was the identity. Pinned in tests/integration/find-hybrid-filter-before-hydrate.test.ts: page two of a `connected` hybrid find matches the pipeline oracle row for row, paging reaches every matching neighbour exactly once, and a `fusion` find's second page is the same page the plain find returns. --- src/brainy.ts | 15 ++++- .../find-hybrid-filter-before-hydrate.test.ts | 55 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/brainy.ts b/src/brainy.ts index ad8bbf90..ffc2d5cd 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -7658,6 +7658,11 @@ export class Brainy implements BrainyInterface { // the hybrid branch, so every other path hydrates unchanged. let finishHybridRow: ((row: Result, pending: Result) => void) | undefined + // Set once the metadata block below has already ranked and CUT the page. + // The tail must not cut it a second time: `offset` has been consumed, and + // re-slicing a `limit`-long page by `offset` returns nothing at all. + let pagedEarly = false + // Handle text-only query (user explicitly wants text search) if (searchMode === 'text' && params.query && params.query.trim() !== '') { results = await this.executeTextSearch(params.query, limit * 2) @@ -7754,6 +7759,7 @@ export class Brainy implements BrainyInterface { const k = offset + limit const order = rankIndicesByScore(results.map(r => r.score), k, true) results = reorderByIndices(results, order).slice(offset, k) + pagedEarly = true // Batch-load entities only for the paginated results (10x faster on GCS). // This is the hydrate-last seam for the deferring paths: a row that @@ -7868,8 +7874,15 @@ export class Brainy implements BrainyInterface { // Efficient pagination - only slice what we need (limit already defined // above), THEN read canonical for the page. Rows that arrived hydrated // pass straight through; a deferred path reads exactly these rows. + // + // A page the metadata block already cut is NOT cut again: it holds the + // rows at [offset, offset+limit) of the ranking, so slicing it by + // `offset` a second time drops the whole page. That is how + // `find({ query, connected, where, offset })` — the shapes that reach + // here after early paging, `connected` and `fusion` — answered [] for + // every page but the first. return await this.hydrateResultPage( - results.slice(finalOffset, finalOffset + limit), + pagedEarly ? results : results.slice(finalOffset, finalOffset + limit), finishHybridRow ) })() diff --git a/tests/integration/find-hybrid-filter-before-hydrate.test.ts b/tests/integration/find-hybrid-filter-before-hydrate.test.ts index 252fe89b..3e74f5d8 100644 --- a/tests/integration/find-hybrid-filter-before-hydrate.test.ts +++ b/tests/integration/find-hybrid-filter-before-hydrate.test.ts @@ -360,6 +360,61 @@ describe('hybrid find: filter before hydrate — the answer is unchanged', () => for (const r of actual) expect(neighbours.has(r.id)).toBe(true) }) + it('hybrid + connected + offset: page two is the page, not an empty answer', async () => { + const params = { + query: QUERY, + connected: { from: anchor, direction: 'out' as const }, + where: { lane: 'alpha' }, + limit: 5, + offset: 5 + } + const expected = await legacyHybridFind(brain as any, params) + expect(expected).toHaveLength(5) + const actual = await brain.find(params as any) + expect(actual.length).toBe(expected.length) + expect(project(actual)).toEqual(expected) + }) + + it('hybrid + connected: paging reaches every matching neighbour exactly once', async () => { + const seen = new Set() + for (let offset = 0; offset < MATCHES; offset += 6) { + const page = await brain.find({ + query: QUERY, + connected: { from: anchor, direction: 'out' as const }, + where: { lane: 'alpha' }, + limit: 6, + offset + } as any) + for (const r of page) { + expect(seen.has(r.id)).toBe(false) + seen.add(r.id) + } + } + // Every row the fused candidate set holds is reachable by paging, and the + // neighbour set is the ceiling. + expect(seen.size).toBeGreaterThanOrEqual(MATCHES) + const neighbours = new Set(matchIds) + for (const id of seen) expect(neighbours.has(id)).toBe(true) + }) + + it('hybrid + fusion + offset: page two is the page', async () => { + const plain = await brain.find({ + query: QUERY, + where: { lane: 'alpha' }, + limit: 5, + offset: 5 + } as any) + const fused = await brain.find({ + query: QUERY, + where: { lane: 'alpha' }, + fusion: 'weighted', + limit: 5, + offset: 5 + } as any) + expect(fused).toHaveLength(plain.length) + expect(fused.map((r) => r.id)).toEqual(plain.map((r) => r.id)) + }) + it('a hydrated hybrid row is shaped exactly as an eagerly-built one', async () => { const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: 8 } as any) const row = rows[0] From bc70c43d0214aca047e925dcd2a4bfee9a860ba9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:07:30 -0700 Subject: [PATCH 097/150] perf(open): a sealed segment the manifest proves is below the bound is never read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every log-authority open asks the fact log one question — is there a fact above the committed pointer? — and answered it by reading and CRC-decoding EVERY segment file the manifest names. MEASURED in production on a 16k-row brain at generation ~478,819: 34-37 seconds inside `generation-store-open-fold` on every open, including the clean one where the answer is always "nothing". The manifest already knows. A sealed segment's `lastGeneration` is written at seal time, and the seal order has been the same since the log was introduced: `rotate()` fsyncs the tail's bytes FIRST ("sealed segments are always fully durable"), builds the entry from the content that fsync covered, and only then flips the manifest — atomically, fsynced, and in the same write re-pointing `tailSegment`, so a sealed file is never appended to again. A crash in that order is safe in the pruning direction: before the manifest write the segment is still the TAIL and is read whole; after it, the entry describes bytes that were already durable. The only later mutation of a sealed segment is open()'s straddle truncation, which removes facts and re-derives the entry from the actual bytes — a recorded bound can drift DOWN with its file, never up. So `lastGeneration = L` proves the file holds no fact above L, and both manifest-direct passes (`peekFactsAbove` and its streaming twin, the recovery fold) now read only the unsealed tail, entries with no numeric `lastGeneration` — legacy or hand-repaired manifests, never prune what you cannot prove — and entries whose recorded maximum is actually above the bound. The open narrates what it read and what it pruned when the log holds more than one segment. Pinned in tests/integration/factlog-open-prune.test.ts, from the log's own counters rather than a clock: a clean reopen over five sealed segments reads exactly the tail (1 of 6) and finds nothing; a real SIGKILLed writer that sealed segments holding facts above the committed pointer has those segments READ, and its peek, its fold stream and its rollback all match the unpruned full scan fact for fact; a manifest entry missing `lastGeneration` is read. --- src/db/factLog.ts | 123 +++++-- tests/integration/factlog-open-prune.test.ts | 360 +++++++++++++++++++ 2 files changed, 462 insertions(+), 21 deletions(-) create mode 100644 tests/integration/factlog-open-prune.test.ts diff --git a/src/db/factLog.ts b/src/db/factLog.ts index ca130454..728be4b1 100644 --- a/src/db/factLog.ts +++ b/src/db/factLog.ts @@ -40,7 +40,10 @@ * The manifest (`_generations/facts/manifest.json`, JSON — forensics stay * terminal-readable) is the single source of truth for the segment SET; * rotation flips it atomically (write-new → fsync → rename) BEFORE the new - * tail's first byte exists, so no segment file is ever unaccounted for. + * tail's first byte exists, so no segment file is ever unaccounted for. Its + * per-segment `firstGeneration`/`lastGeneration` are LOAD-BEARING at open: a + * recovery pass looking for facts above a bound reads only the segments those + * bounds cannot rule out (the prune law — see `segmentsHoldingFactsAbove`). * * ## Mixed-version logs (the v2 live-write cutover) * @@ -689,6 +692,74 @@ function parseSegment( return { facts, validBytes: offset, formatVersion: FACT_LOG_FORMAT_V1 } } +/** + * THE PRUNE LAW — which segment files a pass looking for facts ABOVE + * `committedGeneration` actually has to read, and how many the manifest's own + * recorded bounds took off the table. + * + * A sealed segment's `lastGeneration` is written at SEAL time and never + * mutated upward afterwards ({@link FactLog.rotate}, unchanged since the log + * was introduced): the tail's bytes are fsynced FIRST (`await this.sync()` — + * "sealed segments are always fully durable"), the entry is then built from + * the content that fsync covered, and only then does the manifest flip — + * atomically (tmp+rename) and fsynced — which in the SAME write re-points + * `tailSegment` at a new file, so the sealed file is never appended to again. + * A crash anywhere in that order is safe in the pruning direction: crash + * before the manifest write and the segment is still the TAIL (read whole); + * crash after it and the entry describes bytes that were already durable. The + * only later mutation of a sealed segment is `open()`'s straddle truncation, + * which REMOVES facts and re-derives the entry from the actual bytes — so a + * recorded bound can drift DOWN with its file, never up. + * + * Therefore: `lastGeneration = L` proves the file holds no fact above L, and + * a pass above `committedGeneration >= L` can skip it whole — no read, no + * CRC decode, no msgpack. What the manifest cannot PROVE is never pruned: an + * entry with no numeric `lastGeneration` (a legacy or hand-repaired manifest) + * is read, and the unsealed tail is always read. + * + * This is the difference between an open that costs O(whole fact log) and one + * that costs O(the facts that could matter). MEASURED in production: a 16k-row + * brain at generation ~478,819 paid 34-37s of segment reads and CRC decoding + * in `generation-store-open-fold` on EVERY open — to answer a question whose + * answer, after a clean close, is always "nothing". + */ +function segmentsHoldingFactsAbove( + stored: FactsManifest, + committedGeneration: number +): { files: string[]; pruned: number } { + const files: string[] = [] + let pruned = 0 + for (const entry of stored.segments) { + const last = (entry as Partial).lastGeneration + if (typeof last === 'number' && Number.isFinite(last) && last <= committedGeneration) { + pruned++ + continue + } + files.push(entry.file) + } + if (stored.tailSegment) files.push(stored.tailSegment) + return { files, pruned } +} + +/** + * Say what the open actually read. One line, and only when the log holds more + * than one segment (a single-segment log has nothing to prune and nothing to + * report) — the operator's receipt that the open is paying for the tail, not + * for the whole history. + */ +function narrateAboveScan( + pass: string, + committedGeneration: number, + read: number, + pruned: number +): void { + if (read + pruned <= 1) return + prodLog.narrate( + `[FactLog] ${pass} above generation ${committedGeneration}: ${read} segment(s) read, ` + + `${pruned} pruned of ${read + pruned} (sealed at or below the bound)` + ) +} + /** * The generation fact log. One instance per open store; every method assumes * the single-writer discipline the generation store already enforces (calls @@ -754,22 +825,6 @@ export class FactLog { return this.manifest.brainId !== undefined || this.tailVersion === FACT_LOG_FORMAT_V2 } - /** - * Open the log and reconcile it to committed truth: read the manifest, - * establish the tail's intact content (torn-tail scan), then TRUNCATE any - * fact with `generation > committedGeneration` — those never committed (a - * crash between fact-append and the commit point). After open, the log is - * exactly the committed prefix. - */ - /** - * Read (without truncating) every intact fact ABOVE a generation — the - * log-authority recovery surface: after a crash, facts beyond the - * manifest watermark that survived with valid CRCs are ACKED writes in - * durable-at-ack mode, and the owner REPLAYS them instead of letting - * open() truncate them. Must be called BEFORE open() (it reads the raw - * segments directly; the torn tail's invalid suffix is ignored exactly - * like open() would). - */ /** * STREAMING twin of {@link FactLog.peekFactsAbove} for the recovery fold: * yields facts above the bound one SEGMENT at a time, ascending, without @@ -779,13 +834,18 @@ export class FactLog { * Works manifest-direct (safe before {@link FactLog.open}). Ordering is * structural (segments rotate in order; appends are ordered within one) and * ASSERTED — a violation aborts loudly, never a silent misordered replay. + * + * Reads only the segments that CAN hold a fact above the bound — see + * {@link segmentsHoldingFactsAbove}. A bounded fold above a high checkpoint + * therefore reads its own tail, not the whole history it already proved + * durable. */ async *streamFactsAbove(committedGeneration: number): AsyncGenerator { const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return if (stored.formatVersion !== FACTS_FORMAT_VERSION) return - const files = [...stored.segments.map((s) => s.file)] - if (stored.tailSegment) files.push(stored.tailSegment) + const { files, pruned } = segmentsHoldingFactsAbove(stored, committedGeneration) + narrateAboveScan('recovery fold', committedGeneration, files.length, pruned) let lastGen = committedGeneration for (const file of files) { const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) @@ -807,13 +867,27 @@ export class FactLog { } } + /** + * Read (without truncating) every intact fact ABOVE a generation — the + * log-authority recovery surface: after a crash, facts beyond the + * manifest watermark that survived with valid CRCs are ACKED writes in + * durable-at-ack mode, and the owner REPLAYS them instead of letting + * open() truncate them. Must be called BEFORE open() (it reads the raw + * segments directly; the torn tail's invalid suffix is ignored exactly + * like open() would). + * + * Reads only the segments that CAN hold such a fact — see + * {@link segmentsHoldingFactsAbove}. This runs on EVERY log-authority open, + * including the clean one where the answer is always empty, so the segments + * the manifest already proves irrelevant are never opened at all. + */ async peekFactsAbove(committedGeneration: number): Promise { const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null if (!stored || typeof stored !== 'object' || !Array.isArray(stored.segments)) return [] if (stored.formatVersion !== FACTS_FORMAT_VERSION) return [] const out: CommitFact[] = [] - const files = [...stored.segments.map((s) => s.file)] - if (stored.tailSegment) files.push(stored.tailSegment) + const { files, pruned } = segmentsHoldingFactsAbove(stored, committedGeneration) + narrateAboveScan('above-manifest peek', committedGeneration, files.length, pruned) for (const file of files) { const bytes = await this.storage.readRawBytes(`${FACTS_PREFIX}/${file}`) if (bytes === null) continue @@ -826,6 +900,13 @@ export class FactLog { return out } + /** + * Open the log and reconcile it to committed truth: read the manifest, + * establish the tail's intact content (torn-tail scan), then TRUNCATE any + * fact with `generation > committedGeneration` — those never committed (a + * crash between fact-append and the commit point). After open, the log is + * exactly the committed prefix. + */ async open(committedGeneration: number): Promise { const stored = (await this.storage.readRawObject(FACTS_MANIFEST_PATH)) as FactsManifest | null if (stored && typeof stored === 'object' && Array.isArray(stored.segments)) { diff --git a/tests/integration/factlog-open-prune.test.ts b/tests/integration/factlog-open-prune.test.ts new file mode 100644 index 00000000..223e91f4 --- /dev/null +++ b/tests/integration/factlog-open-prune.test.ts @@ -0,0 +1,360 @@ +/** + * @module tests/integration/factlog-open-prune + * @description THE OPEN READS THE TAIL, NOT THE HISTORY. + * + * Every log-authority open asks the fact log one question — "is there a fact + * above the committed pointer?" — and until this lane existed it answered by + * reading and CRC-decoding EVERY segment file the manifest names. MEASURED in + * production on a 16k-row brain at generation ~478,819: 34-37 seconds inside + * the `generation-store-open-fold` phase, on every open, including the clean + * one where the answer is always "nothing". + * + * The manifest already records each sealed segment's `lastGeneration`, written + * at seal time AFTER the segment's bytes are fsynced and into a manifest that + * is itself written atomically and fsynced — and a sealed file is never + * appended to again (the same manifest flip re-points `tailSegment`). So an + * entry recording `lastGeneration ≤ committed` PROVES its file holds nothing + * above the bound, and the open can skip it whole. + * + * Pinned here, from the log's own counters (the narration line), never a clock: + * + * 1. A clean close and reopen on a log with ≥4 sealed segments reads + * EXACTLY the tail (1 of 6), prunes the rest, and finds nothing. + * 2. A real SIGKILLed process that sealed segments holding facts ABOVE the + * committed pointer: the reopen READS those sealed segments and recovers + * byte-identically to an unpruned open (differential — the same store, + * with the provable field stripped from its manifest, takes the full-scan + * path and must agree fact for fact, before and after `open()`). + * 3. A manifest entry with no `lastGeneration` (legacy, or hand-repaired) is + * READ. Never prune what the manifest cannot prove. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { spawn } from 'node:child_process' +import { + FactLog, + FACTS_MANIFEST_PATH, + type CommitFact, + type FactLogStorage +} from '../../src/db/factLog.js' +import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' + +const REPO_ROOT = process.cwd() +const TSX = path.join(REPO_ROOT, 'node_modules', '.bin', 'tsx') +/** ~1KB frames against a 4KB rotation threshold: ~5 facts per segment. */ +const ROTATE_BYTES = 4096 + +const tmpDirs: string[] = [] +function makeTempDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factlog-prune-')) + tmpDirs.push(dir) + return dir +} + +afterEach(() => { + for (const dir of tmpDirs.splice(0)) { + try { + fs.rmSync(dir, { recursive: true, force: true }) + } catch { + /* best effort */ + } + try { + fs.rmSync(`${dir}.ready.json`, { force: true }) + } catch { + /* best effort */ + } + } +}) + +const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + +/** One ~1KB fact — the padding is what makes rotation cheap to provoke. */ +function fact(generation: number): CommitFact { + return { + generation, + timestamp: 1_700_000_000_000 + generation, + ops: [ + { + kind: 'noun', + id: UUID(generation), + record: { + metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation }, + vector: null + } + } + ] + } +} + +/** + * A deterministic int minter so the log writes the V2 format production + * writes (the prune is a manifest-level decision and never touches segment + * bytes — but the pins should run against the bytes the fleet actually has). + */ +function makeMinter(): (kind: 'noun' | 'verb', id: string) => bigint { + const ints = new Map() + return (kind, id) => { + const key = `${kind}:${id}` + let minted = ints.get(key) + if (minted === undefined) { + minted = BigInt(ints.size + 1) + ints.set(key, minted) + } + return minted + } +} + +/** Open a fact log over a store directory (a fresh adapter each time — this is + * what a reopen actually does). */ +async function openStore(dir: string): Promise<{ storage: any; log: FactLog }> { + const storage: any = new FileSystemStorage(dir) + await storage.init() + const log = new FactLog(storage as FactLogStorage, { rotateBytes: ROTATE_BYTES }) + log.setIntMinter(makeMinter()) + return { storage, log } +} + +/** Build a log of `count` facts (rotating every ~5), left durable, not closed. */ +async function buildLog(dir: string, count: number): Promise { + const { log } = await openStore(dir) + await log.open(0) + for (let g = 1; g <= count; g++) await log.append(fact(g)) + await log.sync() + return log.headGeneration() +} + +/** Capture the narration channel (`prodLog.narrate` → console.warn). */ +async function captureNarration( + fn: () => Promise +): Promise<{ result: T; lines: string[] }> { + const lines: string[] = [] + const original = console.warn + console.warn = ((...args: unknown[]) => { + lines.push(args.map((a) => String(a)).join(' ')) + }) as typeof console.warn + try { + return { result: await fn(), lines } + } finally { + console.warn = original + } +} + +/** The counters the open narrated — the pin's only source of truth for what + * was read (a wall-clock assertion could pass on a warm page cache). */ +function scanCounts(lines: string[]): { read: number; pruned: number; total: number } { + const line = lines.find((l) => l.includes('[FactLog] above-manifest peek above generation')) + if (!line) { + throw new Error(`no peek narration in:\n${lines.join('\n')}`) + } + const match = /(\d+) segment\(s\) read, (\d+) pruned of (\d+)/.exec(line) + if (!match) throw new Error(`unparsable peek narration: ${line}`) + return { read: Number(match[1]), pruned: Number(match[2]), total: Number(match[3]) } +} + +interface SegmentEntryOnDisk { + file: string + firstGeneration: number + lastGeneration?: number + facts: number + bytes: number +} + +async function readManifest(dir: string): Promise<{ + segments: SegmentEntryOnDisk[] + tailSegment: string | null +}> { + const storage: any = new FileSystemStorage(dir) + await storage.init() + return (await storage.readRawObject(FACTS_MANIFEST_PATH)) as any +} + +async function rewriteManifest( + dir: string, + mutate: (manifest: any) => void +): Promise { + const storage: any = new FileSystemStorage(dir) + await storage.init() + const manifest = await storage.readRawObject(FACTS_MANIFEST_PATH) + mutate(manifest) + await storage.writeRawObject(FACTS_MANIFEST_PATH, manifest) + await storage.syncRawObjects([FACTS_MANIFEST_PATH]) +} + +/** Every fact the log holds, in order — the recovered state, read back. */ +async function allFacts(log: FactLog): Promise { + const out: CommitFact[] = [] + const handle = log.scanFacts() + for await (const batch of handle.batches()) out.push(...batch.facts) + return out +} + +describe('fact log — the open reads only the segments that can hold facts above the bound', () => { + it('a clean close + reopen over ≥4 sealed segments reads exactly the tail and finds nothing', async () => { + const dir = makeTempDir() + const head = await buildLog(dir, 30) + + const manifest = await readManifest(dir) + expect(manifest.segments.length).toBeGreaterThanOrEqual(4) // the fixture is real + expect(manifest.tailSegment).not.toBeNull() + + // The reopen: a clean close means committed === the log's head. + const { log } = await openStore(dir) + const { result: orphans, lines } = await captureNarration(() => log.peekFactsAbove(head)) + + expect(orphans).toEqual([]) // the fold finds nothing, as it always does after a clean close + const counts = scanCounts(lines) + expect(counts.read).toBe(1) // EXACTLY the tail + expect(counts.total).toBe(manifest.segments.length + 1) + expect(counts.pruned).toBe(manifest.segments.length) + + // And the reconciling open still lands on the same committed prefix. + await log.open(head) + expect(log.headGeneration()).toBe(head) + expect((await allFacts(log)).map((f) => f.generation)).toEqual( + Array.from({ length: head }, (_, i) => i + 1) + ) + }) + + it('a manifest entry with no lastGeneration is READ — never prune what you cannot prove', async () => { + const dir = makeTempDir() + const head = await buildLog(dir, 30) + const before = await readManifest(dir) + expect(before.segments.length).toBeGreaterThanOrEqual(4) + + // A legacy/hand-repaired entry: the field the prune needs is simply absent. + await rewriteManifest(dir, (m) => { + delete m.segments[0].lastGeneration + }) + + const { log } = await openStore(dir) + const { result: orphans, lines } = await captureNarration(() => log.peekFactsAbove(head)) + + expect(orphans).toEqual([]) // still nothing above the bound — it was READ to find out + const counts = scanCounts(lines) + expect(counts.read).toBe(2) // the unprovable entry + the tail + expect(counts.pruned).toBe(before.segments.length - 1) + expect(counts.total).toBe(before.segments.length + 1) + }) + + it( + 'a SIGKILLed writer that sealed segments above the committed pointer recovers identically to an unpruned open', + async () => { + const dir = makeTempDir() + const readyPath = `${dir}.ready.json` + // A real process death: the child fsyncs its segments, records what it + // reached, and SIGKILLs ITSELF — no close, no unwind, no chance to tidy. + const script = ` + import * as fs from 'node:fs' + import { FactLog } from ${JSON.stringify(path.join(REPO_ROOT, 'src', 'db', 'factLog.ts'))} + import { FileSystemStorage } from ${JSON.stringify(path.join(REPO_ROOT, 'src', 'storage', 'adapters', 'fileSystemStorage.ts'))} + const UUID = (n) => '00000000-0000-4000-8000-' + String(n).padStart(12, '0') + const fact = (g) => ({ + generation: g, + timestamp: 1700000000000 + g, + ops: [{ kind: 'noun', id: UUID(g), record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g }, vector: null } }] + }) + const ints = new Map() + const storage = new FileSystemStorage(${JSON.stringify(dir)}) + await storage.init() + const log = new FactLog(storage, { rotateBytes: ${ROTATE_BYTES} }) + log.setIntMinter((kind, id) => { + const key = kind + ':' + id + if (!ints.has(key)) ints.set(key, BigInt(ints.size + 1)) + return ints.get(key) + }) + await log.open(0) + for (let g = 1; g <= 30; g++) await log.append(fact(g)) + await log.sync() + fs.writeFileSync(${JSON.stringify(readyPath)}, JSON.stringify({ head: log.headGeneration() })) + process.kill(process.pid, 'SIGKILL') + ` + const scriptPath = path.join(dir, 'crash-writer.mts') + fs.writeFileSync(scriptPath, script) + const child = spawn(TSX, [scriptPath], { cwd: REPO_ROOT, stdio: ['ignore', 'pipe', 'pipe'] }) + let output = '' + child.stdout.on('data', (d) => { output += String(d) }) + child.stderr.on('data', (d) => { output += String(d) }) + const exit = await new Promise<{ code: number | null; signal: string | null }>((resolve) => + child.on('exit', (code, signal) => resolve({ code, signal })) + ) + if (!fs.existsSync(readyPath)) { + throw new Error(`the crash writer never reached its kill point:\n${output}`) + } + // Death, not a shutdown: no close(), no unwind, no orderly exit code. + expect(exit.signal ?? `code ${exit.code}`).not.toBe('code 0') + const head = JSON.parse(fs.readFileSync(readyPath, 'utf8')).head as number + expect(head).toBe(30) + + // The committed pointer the survivor comes back on: mid-log, so sealed + // segments hold facts ABOVE it — the exact shape the prune must not skip. + const committed = 12 + const manifest = await readManifest(dir) + const straddling = manifest.segments.filter( + (s) => s.firstGeneration <= committed && (s.lastGeneration ?? 0) > committed + ) + const entirelyAbove = manifest.segments.filter((s) => s.firstGeneration > committed) + expect(straddling.length).toBeGreaterThanOrEqual(1) + expect(entirelyAbove.length).toBeGreaterThanOrEqual(1) + + // THE DIFFERENTIAL. The unpruned answer, through the SAME code on the + // SAME bytes: a peek above generation 0 can prune nothing (no sealed + // segment ends at or below 0), so it reads every segment file and + // decodes every frame — exactly what this open used to do — and its + // facts above the pointer are what the fold is entitled to replay. + const { log } = await openStore(dir) + const { result: fullScan, lines: fullLines } = await captureNarration(() => + log.peekFactsAbove(0) + ) + expect(scanCounts(fullLines)).toEqual({ + read: manifest.segments.length + 1, + pruned: 0, + total: manifest.segments.length + 1 + }) + const unprunedAnswer = fullScan.filter((f) => f.generation > committed) + + const { result: prunedAnswer, lines } = await captureNarration(() => + log.peekFactsAbove(committed) + ) + + // The sealed segments above the bound were READ, not skipped. + const counts = scanCounts(lines) + expect(counts.read).toBe(straddling.length + entirelyAbove.length + 1) + expect(counts.pruned).toBe(manifest.segments.length - straddling.length - entirelyAbove.length) + expect(counts.pruned).toBeGreaterThan(0) // the prune did engage, and was still right + expect(prunedAnswer.map((f) => f.generation)).toEqual( + Array.from({ length: head - committed }, (_, i) => committed + 1 + i) + ) + // Facts that live in a SEALED segment (not the tail) came back. + expect(prunedAnswer.some((f) => f.generation <= (straddling[0].lastGeneration ?? 0))).toBe( + true + ) + // Fact for fact, the pruned answer IS the unpruned answer — so whatever + // the recovery replays, it replays identically. + expect(prunedAnswer).toEqual(unprunedAnswer) + + // The fold's streaming twin (the unclean-open path) agrees too. + const streamed: CommitFact[] = [] + for await (const batch of log.streamFactsAbove(committed)) streamed.push(...batch) + expect(streamed).toEqual(unprunedAnswer) + + // And the reconciling open rolls back exactly as it always did: the two + // never-committed sealed segments dropped, the straddling one cut, the + // tail truncated — the log left as the committed prefix. + await log.open(committed) + expect(log.headGeneration()).toBe(committed) + expect((await allFacts(log)).map((f) => f.generation)).toEqual( + Array.from({ length: committed }, (_, i) => i + 1) + ) + const after = await readManifest(dir) + expect(after.segments.map((s) => s.file)).toEqual( + manifest.segments + .filter((s) => s.firstGeneration <= committed) + .map((s) => s.file) + ) + expect(after.segments[after.segments.length - 1].lastGeneration).toBe(committed) + }, + 120_000 + ) +}) From 15d4f65dcf65379b27dca7f0c7e6e2297cdccab5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:08:34 -0700 Subject: [PATCH 098/150] perf(open): the pending-embed fold is bounded by a checkpoint of the SET, not an empty-only mark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The low-water mark shipped in 10.4.9 can only be written when the pending set is EMPTY, because it carries no set — it means "everything at or below G is consumed". A brain holding even one id that never lands (an embed that keeps failing, a data-less row reaped in memory only and re-folded every open) never drains, so it never writes a mark, so the bound never engaged on exactly the brains whose fold is expensive: `recover-pending-embeds` re-read the WHOLE fact log at every open, on the open's foreground. _system/pending_embeds_checkpoint.json carries the set: { generation, pending, writtenAt } = "as of durable generation G the pending set was exactly this list". Open seeds the set from the list and scans from G + 1, so the fold is O(facts since G) whether or not the set ever drains. Measured on a 301-row brain with one stuck id: 302 facts read before, 0 after; at 601 rows, 602 before, 0 after — same pending set both ways. THE DURABILITY LAW, by construction. A checkpoint at head H taken while the facts up to H are still buffered would be read back after a crash that truncated the tail: an `embed.landed` in a truncated fact would be gone from the log while the checkpoint still recorded its id as landed, and its landing vector went with the fact — a LOST VECTOR. So a capture is refused unless `0 < head <= committed`, the manifest watermark below which FactLog.open() never truncates and which the group-commit flush only advances after fsyncing the log. The (generation, set) pair is taken in one synchronous instant with no await between reading the generations and snapshotting the set. The one remaining asymmetry runs the safe way: an id enqueued in memory whose marker lands at G+1 is captured as pending at G — one idempotent re-embed, never a loss. Written at clean close (inside closeDurableSteps, after the generation store's own close flushed the log and advanced the manifest), at drain-to-empty, and on a cadence of max(64, ceil(|pending| / 64)) transitions while open — an interval that holds the mechanism's amortized cost at <= 64 ids written per transition however large the backlog grows, so the cure cannot reintroduce the defect class it fixes. No timer, no knob. The debt stays armed across attempts the durability law refuses, so a write burst does not skip a checkpoint, it defers it. Degradation is loud and always toward a LONGER scan: a torn checkpoint throws typed on read (the adapter's tmp+rename write means it can never parse into a partial list) and a malformed one is refused whole, both falling back to the low-water mark — still written, still read — and then to generation 1. The fold narrates which bound applied and how many facts it read, on every open, so a bound that stops engaging is visible instead of silent. The worker's orphan reap splits: a row that is GONE clears durably (its tombstone is in the log, or its create never was), while a present-but data-less row keeps clearing in memory only and is carried in the checkpoint list, so the bounded fold and a full fold from generation 1 agree exactly. The crash-recovery contract is unchanged: the fold stays on the open's foreground, markers re-armed when open() returns. --- src/brainy.ts | 416 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 390 insertions(+), 26 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index ffc2d5cd..7568a6f3 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -776,6 +776,50 @@ export class Brainy implements BrainyInterface { private _pendingEmbedIds = new Set() private _embedWorkerFlight: Promise | null = null + /** + * Ids cleared from {@link _pendingEmbedIds} with NO durable disarming record + * behind them — today exactly one case: a pending row that still EXISTS but + * carries no embeddable data, which the worker reaps in memory only. The log + * still says those ids are pending, so the pending-embed CHECKPOINT must + * carry them: the checkpoint's contract is "as of generation G the LOG's + * pending set was exactly this list", and a checkpoint that quietly dropped + * an id the log still arms would make the bounded fold disagree with a full + * fold from generation 1 — the one divergence that could lose a vector. + * Bounded by the number of such rows; an id leaves when it is re-enqueued or + * durably disarmed. + */ + private _pendingEmbedUndurableClears = new Set() + + /** + * Pending-set transitions (enqueue/clear) since the last checkpoint attempt — + * the checkpoint CADENCE. One mechanism, one hardcoded default, no knob and + * no timer (nothing to leave running after close). + */ + private _pendingEmbedCheckpointTransitions = 0 + + /** + * A checkpoint is OWED: the cadence came due (or the set drained) and no + * write has satisfied it yet. It stays armed across attempts the durability + * law refuses, so the next transition that CAN be checkpointed is. + */ + private _pendingEmbedCheckpointDue = false + + /** Single-flight guard for the fire-and-forget checkpoint write. */ + private _pendingEmbedCheckpointFlight: Promise | null = null + + /** + * What the last pending-embed recovery fold actually did — the bound it + * used, where it started, and how many facts it read. The narration's + * source, and the accounting a pin reads instead of a clock. + */ + private _pendingEmbedFoldReport: { + bound: 'checkpoint' | 'low-water' | 'genesis' + fromGeneration: number + factsScanned: number + seeded: number + pending: number + } | null = null + // OPEN-PATH FIX: the background embedding-engine warm kicked off (never // awaited) by `performInit()` when `eagerEmbeddings` resolves true. Stored // for observability only — `embed()`/`embeddingManager.embed()` already @@ -2434,6 +2478,47 @@ export class Brainy implements BrainyInterface { */ private static readonly PENDING_EMBED_LOWWATER_PATH = '_system/pending_embeds_lowwater.json' + /** + * Storage-root-relative path of the pending-embed CHECKPOINT: + * `{ generation, pending: string[], writtenAt }` — "as of durable generation + * G the pending set was exactly this list". Open seeds the set from `pending` + * and scans the log from `G + 1`, so the fold costs O(facts since G) + * REGARDLESS of whether the set ever drains. + * + * WHY IT REPLACES THE EMPTY-ONLY MARK AS THE BOUND. The low-water mark + * ({@link PENDING_EMBED_LOWWATER_PATH}) can only be written when the pending + * set is EMPTY, because it carries no set — it means "everything at or below + * G is consumed". A brain holding even ONE id that never lands (an embed that + * keeps failing; a row reaped in memory only and re-folded every open) never + * drains, so it never writes a mark, so the bound never engages on exactly + * the brains whose fold is expensive: every open re-reads the whole log. The + * checkpoint carries the set, so it needs no drain. + * + * The mark is still written and still read as the FALLBACK bound (a + * checkpoint that is absent, torn, or malformed degrades to it, and then to + * generation 1). Correctness over cost in every degradation: a stale or + * missing checkpoint only lengthens the scan. + */ + private static readonly PENDING_EMBED_CHECKPOINT_PATH = '_system/pending_embeds_checkpoint.json' + + /** + * Checkpoint CADENCE BASE: attempt a checkpoint every N pending-set + * transitions (enqueues + clears) while the brain is open, on top of the + * drain-to-empty and clean-close writes. Hardcoded 90th-percentile default, + * no knob, no timer: 64 transitions is far below the cost of the fold it + * bounds and far above the per-write noise floor. An attempt that cannot + * satisfy the durability law is SKIPPED, not forced — the next transition + * retries. + * + * The interval ADAPTS to the one signal that matters, the backlog's own + * size, because a checkpoint writes the WHOLE pending list: the interval is + * `max(64, ceil(|pending| / 64))`, which holds the amortized cost of the + * mechanism at ≤ 64 ids written per transition NO MATTER how large the + * backlog grows. A term that scales with the store rather than with the + * work is exactly the defect class this file is fixing; it must not be + * reintroduced by the cure. + */ + private static readonly PENDING_EMBED_CHECKPOINT_EVERY = 64 /** * @description Mark a deferred embed pending (MT5): the id joins the @@ -2448,6 +2533,9 @@ export class Brainy implements BrainyInterface { */ private enqueuePendingEmbed(id: string): FactMarkerRecord { this._pendingEmbedIds.add(id) + // Re-armed for real: any earlier in-memory-only clear is superseded. + this._pendingEmbedUndurableClears.delete(id) + this.noteEmbedCheckpointCadence() return { type: 'embed.pending', id, enqueuedAt: Date.now() } } @@ -2458,11 +2546,27 @@ export class Brainy implements BrainyInterface { * fact) — the recovery fold consumes those; nothing here touches storage. * One honest residue: a pending row whose entity still exists but carries * no data is reaped in memory only, so it re-folds at the next open and - * is re-reaped there — a bounded no-op, never a lost vector. + * is re-reaped there — a bounded no-op, never a lost vector. That residue + * is the ONLY `durability: 'in-memory-only'` caller, and the checkpoint + * keeps carrying those ids so the bounded fold and a full fold from + * generation 1 agree exactly (see {@link _pendingEmbedUndurableClears}). + * + * @param id - The pending id to clear. + * @param durability - `'durable'` (default) when a record in the log at or + * below the current head disarms this id (an `embed.landed` riding the + * landing or unvector commit, or the row's tombstone — including the row + * simply not being there any more); `'in-memory-only'` when nothing in the + * log says so. */ - private clearPendingEmbed(id: string): void { + private clearPendingEmbed( + id: string, + durability: 'durable' | 'in-memory-only' = 'durable' + ): void { this._pendingEmbedIds.delete(id) + if (durability === 'in-memory-only') this._pendingEmbedUndurableClears.add(id) + else this._pendingEmbedUndurableClears.delete(id) if (this._pendingEmbedIds.size === 0) this.maybeWriteEmbedLowWater() + this.noteEmbedCheckpointCadence() } /** @@ -2498,6 +2602,223 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Capture a pending-embed checkpoint, or refuse. + * + * THE DURABILITY LAW, satisfied by construction. The checkpoint asserts "as + * of generation G the log's pending set was exactly this list", and the next + * open TRUSTS it: it seeds the set and never reads a fact at or below G + * again. So a checkpoint may only be taken at a G whose facts are DURABLE. + * A checkpoint taken at head H while the facts up to H are still buffered + * would be read back after a crash that truncated the tail — and an + * `embed.landed` in a truncated fact would be gone from the log while the + * checkpoint still recorded its id as landed. The row's landing vector went + * with the truncated fact, so nothing would ever re-arm it: A LOST VECTOR. + * + * The gate is therefore `0 < head ≤ committed`. `committed` is the + * generation manifest's watermark — the point the store's own recovery + * treats as truth, and the point below which `FactLog.open()` never + * truncates — and the group-commit flush fsyncs the log BEFORE advancing it + * (see `GenerationStore.flushPendingSingleOps`). So every fact at or below + * `head` is fsynced and survives the crash exactly as the checkpoint + * describes it. Anything else (a head above the manifest, no log, no + * generation yet, a read-only or closed brain) REFUSES: skipping a + * checkpoint costs a longer scan next open, never a marker. + * + * The snapshot is taken SYNCHRONOUSLY with reading the two generations — no + * `await` between them — so no commit and no worker step can slip between + * "the generation I am about to claim" and "the set I claim for it". + * + * The one asymmetry, deliberately in the safe direction: an id whose + * `embed.pending` record has not been appended yet (enqueued in memory, its + * commit still in flight) is captured as pending at G although its marker + * will land at G+1 or later. Over-stating pending costs one idempotent + * re-embed attempt; under-stating it is the shape that loses a vector, and + * cannot happen — every clear either rides a durable record at or below the + * head, or is carried in {@link _pendingEmbedUndurableClears}. + * + * @returns The checkpoint payload, or `null` when this instant cannot host + * one. + */ + private captureEmbedCheckpoint(): { generation: number; pending: string[] } | null { + if (this.isReadOnly || this.closed) return null + const store = this.generationStore + if (!store) return null + const log = store.getFactLog() + if (!log) return null + // --- ONE SYNCHRONOUS INSTANT: no await until the return. --- + const generation = log.headGeneration() + const committed = store.committedGeneration() + if (!(generation > 0) || generation > committed) return null + const pending = new Set(this._pendingEmbedIds) + for (const id of this._pendingEmbedUndurableClears) pending.add(id) + // --- end of the synchronous instant. --- + return { generation, pending: [...pending] } + } + + /** + * @description Fire-and-forget checkpoint write, single-flight: a burst of + * transitions never stacks writes, and because each attempt captures + * immediately before it writes, the file always ends up holding the most + * recently captured (generation, set) PAIR — and every such pair is + * independently true, so even an out-of-order landing is safe. + * {@link closeDurableSteps} awaits the flight before taking the final one. + */ + private maybeWriteEmbedCheckpoint(): void { + if (this._pendingEmbedCheckpointFlight) return + this._pendingEmbedCheckpointFlight = this.writeEmbedCheckpoint() + .then((wrote) => { + if (wrote) { + this._pendingEmbedCheckpointDue = false + this._pendingEmbedCheckpointTransitions = 0 + } + }) + .finally(() => { + this._pendingEmbedCheckpointFlight = null + }) + } + + /** + * The awaitable core of {@link maybeWriteEmbedCheckpoint}. + * @returns `true` when a checkpoint was actually written. + */ + private async writeEmbedCheckpoint(): Promise { + const snapshot = this.captureEmbedCheckpoint() + if (!snapshot) return false + try { + // Atomic on disk: the filesystem adapter's writeRawObject is tmp+rename + // (see BaseStorage.writeRawObject), so a crash mid-write leaves either + // the previous checkpoint or the new one — never a spliced file. And a + // file that IS unreadable (a torn gzip, invalid JSON) throws typed on + // read and degrades to the fallback bound; it can never parse into a + // partial `pending` list. + // + // The file is NOT separately fsynced, and does not need to be: losing + // the rename to a power cut leaves the PREVIOUS checkpoint (or none), + // which only lengthens the next scan. The invariant that matters is the + // other direction — a checkpoint that IS visible names a generation + // whose facts are durable — and that is established by the capture gate + // above, not by this write. + await this.storage.writeRawObject(Brainy.PENDING_EMBED_CHECKPOINT_PATH, { + generation: snapshot.generation, + pending: snapshot.pending, + writtenAt: Date.now() + }) + return true + } catch (err) { + prodLog.warn( + `[Brainy] pending-embed checkpoint write failed at generation ` + + `${snapshot.generation}: ${(err as Error).message} — the next open scans ` + + `from the previous checkpoint` + ) + return false + } + } + + /** + * @description The checkpoint cadence tick: count one pending-set transition + * and OWE a checkpoint every {@link PENDING_EMBED_CHECKPOINT_EVERY} + * transitions, plus on every drain to empty. The debt stays armed across + * attempts the durability law refuses — during a write burst the log head + * legitimately runs ahead of the manifest, so the first attempt often cannot + * be taken — and the next transition retries it. An active brain therefore + * checkpoints steadily without ever forcing a flush; an idle one relies on + * its clean close. No timer is involved, so nothing survives close(). + */ + private noteEmbedCheckpointCadence(): void { + if (this.isReadOnly || this.closed) return + this._pendingEmbedCheckpointTransitions++ + const listed = this._pendingEmbedIds.size + this._pendingEmbedUndurableClears.size + const every = Math.max( + Brainy.PENDING_EMBED_CHECKPOINT_EVERY, + Math.ceil(listed / Brainy.PENDING_EMBED_CHECKPOINT_EVERY) + ) + if ( + this._pendingEmbedIds.size === 0 || + this._pendingEmbedCheckpointTransitions >= every + ) { + this._pendingEmbedCheckpointDue = true + } + if (this._pendingEmbedCheckpointDue) this.maybeWriteEmbedCheckpoint() + } + + /** + * @description Resolve the pending-embed fold's BOUND: the checkpoint first + * (a set plus a generation), then the legacy low-water mark (a generation + * only), then genesis. Every degradation is loud and lengthens the scan + * rather than shortening it — a bound that could skip a marker is never + * derived from a value this method could not fully validate. + * @returns The bound's name, the first generation to scan, and the ids to + * seed the pending set with. + */ + private async readPendingEmbedBound(): Promise<{ + bound: 'checkpoint' | 'low-water' | 'genesis' + fromGeneration: number + seeded: string[] + }> { + let checkpointRejected: string | null = null + try { + const raw = await this.storage.readRawObject(Brainy.PENDING_EMBED_CHECKPOINT_PATH) + if (raw !== null && raw !== undefined) { + const parsed = Brainy.parsePendingEmbedCheckpoint(raw) + if (parsed) { + return { + bound: 'checkpoint', + fromGeneration: parsed.generation + 1, + seeded: parsed.pending + } + } + checkpointRejected = 'its shape is not { generation: number > 0, pending: string[] }' + } + } catch (err) { + // A real storage fault (EIO/EACCES/…). Corruption never lands here: the + // adapter maps a torn raw object to `null` AFTER logging it as a + // production error, so a torn checkpoint arrives as "absent" — loud at + // the adapter, and bounded here by the fallback below. + checkpointRejected = `reading it failed: ${(err as Error).message}` + } + if (checkpointRejected !== null) { + prodLog.warn( + `[Brainy] pending-embed checkpoint REFUSED (${checkpointRejected}) — falling back ` + + `to the low-water mark, else a full fold from generation 1` + ) + } + + try { + const mark = (await this.storage.readRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH)) as { + generation?: number + } | null + if (mark && typeof mark.generation === 'number' && mark.generation > 0) { + return { bound: 'low-water', fromGeneration: mark.generation + 1, seeded: [] } + } + } catch { + // No mark (or unreadable): scan from 1 — correctness over cost. + } + return { bound: 'genesis', fromGeneration: 1, seeded: [] } + } + + /** + * @description Validate a raw checkpoint object STRICTLY. Anything that is + * not exactly `{ generation: integer > 0, pending: string[] }` is refused + * whole — a partially-usable checkpoint is the one shape that could seed a + * short pending set behind a high bound, which is how a vector is lost. + * @param raw - The object read back from storage. + * @returns The validated checkpoint, or `null`. + */ + private static parsePendingEmbedCheckpoint( + raw: unknown + ): { generation: number; pending: string[] } | null { + if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) return null + const { generation, pending } = raw as { generation?: unknown; pending?: unknown } + if (typeof generation !== 'number' || !Number.isSafeInteger(generation) || generation <= 0) { + return null + } + if (!Array.isArray(pending) || pending.some((id) => typeof id !== 'string' || id === '')) { + return null + } + return { generation, pending: pending as string[] } + } + /** * @description Rebuild the pending-embed set by REPLAYING the generation * log's marker records (recovery = replay, not listing): `embed.pending` @@ -2506,14 +2827,22 @@ export class Brainy implements BrainyInterface { * survives the fold is exactly the set of acknowledged deferred writes * whose vectors have not landed. * - * BOUND: the scan starts at the advisory low-water mark - * ({@link Brainy.PENDING_EMBED_LOWWATER_PATH}) — the log head at which the - * pending set last drained to empty — so a settled brain reads only the - * facts since then, not its whole history. Without a mark (first open - * after upgrade) it scans from generation 1, once; a stale-low mark costs - * a longer scan, never a marker. The fold stays on the open's foreground — - * the crash-recovery contract pins that a reopened brain has its markers - * re-armed when open() returns — and the mark is what makes that cheap. + * BOUND: the scan starts after the pending-embed CHECKPOINT + * ({@link Brainy.PENDING_EMBED_CHECKPOINT_PATH}) — "as of durable generation + * G the pending set was exactly this list" — so the fold seeds the set from + * that list and reads only the facts after G. O(delta) whether or not the + * set ever drains, which is the whole point: the previous bound, the + * empty-only low-water mark, could not be written at all by a brain holding + * one id that never lands, so those brains re-read their whole log at every + * open. The mark remains the FALLBACK bound (checkpoint absent, torn, or + * malformed), and generation 1 the fallback below that — a brain opened for + * the first time after this change has neither a checkpoint nor, if it never + * drained, a mark, so it pays one full fold and writes a checkpoint on the + * way out. A stale bound costs a longer scan, never a marker. The fold stays + * on the open's foreground — the crash-recovery contract pins that a + * reopened brain has its markers re-armed when open() returns — and the + * bound is what makes that cheap. What it did (bound, start, facts read) is + * narrated and kept in {@link _pendingEmbedFoldReport}. * It is SKIPPED WHOLESALE when the log has never had a v2 tail * ({@link FactLog.hasV2History} — v1 facts cannot carry marker records), * so pre-cutover brains pay nothing; on a mixed log the scan still reads @@ -2526,20 +2855,13 @@ export class Brainy implements BrainyInterface { private async recoverPendingEmbedsFromLog(): Promise { const log = this.generationStore.getFactLog() if (!log || !log.hasV2History()) return - let fromGeneration = 1 - try { - const mark = (await this.storage.readRawObject(Brainy.PENDING_EMBED_LOWWATER_PATH)) as { - generation?: number - } | null - if (mark && typeof mark.generation === 'number' && mark.generation > 0) { - fromGeneration = mark.generation + 1 - } - } catch { - // No mark (or unreadable): scan from 1 — correctness over cost. - } + const { bound, fromGeneration, seeded } = await this.readPendingEmbedBound() + for (const id of seeded) this._pendingEmbedIds.add(id) + let factsScanned = 0 const scan = log.scanFacts({ fromGeneration }) for await (const batch of scan.batches()) { for (const fact of batch.facts) { + factsScanned++ for (const record of fact.records ?? []) { if (record.type === 'embed.pending') { this._pendingEmbedIds.add(record.id) @@ -2554,6 +2876,21 @@ export class Brainy implements BrainyInterface { } } } + this._pendingEmbedFoldReport = { + bound, + fromGeneration, + factsScanned, + seeded: seeded.length, + pending: this._pendingEmbedIds.size + } + // The narration channel: an operator is entitled to hear which bound + // applied and what it cost, on every open — that is how a bound that + // silently stopped engaging (the defect this replaced) becomes visible. + prodLog.narrate( + `[Brainy] pending-embed fold: ${bound} bound → scanned ${factsScanned} fact(s) ` + + `from generation ${fromGeneration}, seeded ${seeded.length} id(s), ` + + `${this._pendingEmbedIds.size} pending` + ) } /** @@ -2645,11 +2982,23 @@ export class Brainy implements BrainyInterface { for (const id of batch) { try { const entity = await this.get(id, { includeVectors: true }) - if (!entity || entity.data === undefined || entity.data === null) { - // Orphan reap: a deleted row's tombstone fact durably disarms the - // marker at the next recovery fold; a data-less-but-present row - // (edge case) re-folds and re-reaps — bounded, never a lost vector. - this.clearPendingEmbed(id) + if (!entity) { + // The row is GONE. Either it was deleted — its tombstone fact + // durably disarms the marker, at or below the head, exactly as the + // fold reads it — or its create never became durable, in which case + // the log carries no `embed.pending` for it either. Both are durable + // clears: a full fold from generation 1 reaches the same answer. + this.clearPendingEmbed(id, 'durable') + continue + } + if (entity.data === undefined || entity.data === null) { + // Orphan reap, IN MEMORY ONLY: a data-less-but-present row (edge + // case) has nothing to embed, but no record in the log says so, so + // the fold would re-arm it. Cleared here and carried in the + // checkpoint (see clearPendingEmbed) — it re-folds and re-reaps at + // the next open exactly as before: bounded, never a lost vector, + // and never a checkpoint that disagrees with the log. + this.clearPendingEmbed(id, 'in-memory-only') continue } // Hang guard: a wedged embedder must not block every later pending @@ -19982,6 +20331,21 @@ export class Brainy implements BrainyInterface { await this.stampEntityTree() } + // Phase 1c: the pending-embed CHECKPOINT — placed HERE and not earlier + // because this is the first point in the close where the durability law it + // must satisfy actually holds: `generationStore.close()` (in Phase 1 above) + // flushed the pending single-op tier, which fsyncs the fact log and then + // advances the manifest, so `head === committed` and every fact the + // checkpoint's generation covers is durable. Taken even when the set is + // NOT empty — that is the whole difference from the low-water mark, and it + // is what makes the next open's fold O(facts since this close) on a brain + // whose pending set never drains. Awaits any in-flight cadence write first + // so the last write to the file is this one. + if (!this.isReadOnly) { + await this._pendingEmbedCheckpointFlight?.catch(() => {}) + await this.writeEmbedCheckpoint() + } + // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 await Promise.all([ From 1fb51093511998db05d44423810baee37aa3e8b5 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:08:43 -0700 Subject: [PATCH 099/150] =?UTF-8?q?test(open):=20pin=20the=20pending-embed?= =?UTF-8?q?=20checkpoint=20=E2=80=94=20stuck=20id,=20crash=20matrix,=20tor?= =?UTF-8?q?n=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four things, none of them a clock: 1. A brain with one permanently-stuck pending id, closed cleanly and reopened, scans ONLY the facts after the checkpoint — read from the fold's own accounting. The same fixture pins the DEFECT it cures: no low-water mark exists on that brain, because it never drained, so nothing could have shortened its fold. A second row proves the bound stays O(delta) across repeated opens while the id is still stuck. 2. A crash matrix in a REAL child process (detached group, SIGKILL, no close), following writer-lock-clean-close's pattern: killed before any checkpoint was written, killed after one with an embed landed and flushed above it, and killed after one with an UN-FLUSHED tail. The invariant in every row is differential — the checkpoint-bounded fold the reopened brain actually ran equals a full fold from generation 1 over the same recovered log. 3. A torn checkpoint (bytes that are neither gzip nor JSON) falls back loudly — the adapter's torn-record gauge and production error, plus the fold's own narration of the bound it used — and still recovers the marker from the log. A well-formed but shape-invalid checkpoint is refused WHOLE: trusting its generation while ignoring its list is the one shape that could bound a scan behind a set that was never recovered. 4. The existing low-water pins pass unchanged — the mark is still written and still read, now as the fallback bound beneath the checkpoint. --- .../pending-embed-checkpoint.test.ts | 547 ++++++++++++++++++ 1 file changed, 547 insertions(+) create mode 100644 tests/integration/pending-embed-checkpoint.test.ts diff --git a/tests/integration/pending-embed-checkpoint.test.ts b/tests/integration/pending-embed-checkpoint.test.ts new file mode 100644 index 00000000..1cf3ec2c --- /dev/null +++ b/tests/integration/pending-embed-checkpoint.test.ts @@ -0,0 +1,547 @@ +/** + * @module tests/integration/pending-embed-checkpoint + * @description THE PENDING-EMBED CHECKPOINT — the bound that engages on the + * brains that need it. + * + * 10.4.9 bounded the open-path `recover-pending-embeds` fold with a LOW-WATER + * MARK: the log head at which the pending set last drained to EMPTY. That mark + * carries no set, so it can only be written when the set is empty — and a brain + * holding even ONE id that never lands (an embed that keeps failing, a worker + * that never gets to it, a row reaped in memory only and re-folded every open) + * never drains, therefore never writes a mark, therefore re-reads its WHOLE + * fact log on every single open. The bound was absent from exactly the brains + * whose fold is expensive: a silent scaling defect. + * + * The cure is a CHECKPOINT of the pending set — + * `_system/pending_embeds_checkpoint.json` = `{ generation, pending, writtenAt }`, + * meaning "as of durable generation G the pending set was exactly this list". + * Open seeds the set from `pending` and scans only from `G + 1`, so the fold is + * O(facts since G) whether or not the set ever drains. + * + * What this suite pins: + * 1. A brain with one permanently-stuck pending id, closed cleanly and + * reopened, scans ONLY the facts after the checkpoint — asserted from the + * fold's own accounting, never a clock. The same fixture pins the DEFECT: + * no low-water mark exists on that brain, because it never drained. + * 2. A crash matrix in a REAL child process (SIGKILL, no close), for kills + * before a checkpoint write, after one with embeds landed and flushed + * after it, and after one with an UN-FLUSHED tail at the moment of death. + * The invariant in every row is differential: the checkpoint-bounded fold + * the reopened brain actually ran ≡ a full fold from generation 1 over the + * same recovered log. + * 3. A torn checkpoint falls back — loudly (the adapter's torn-record gauge + * plus the fold's own narration of which bound applied) and correctly. + * 4. The existing low-water pins keep passing unchanged + * (`pending-embed-low-water.test.ts`): the mark is still written and is + * still read, now as the FALLBACK bound beneath the checkpoint. + * + * The crash-recovery contract is untouched: the fold runs on the open's + * foreground, so a reopened brain has its markers re-armed when open() returns. + */ +import { describe, it, expect, afterEach } from 'vitest' +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs' +import { spawn } from 'node:child_process' +import { gunzipSync } from 'node:zlib' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { getTornRecordGauge } from '../../src/storage/tornRecordError.js' + +const CHECKPOINT_PATH = '_system/pending_embeds_checkpoint.json' +const LOWWATER_PATH = '_system/pending_embeds_lowwater.json' +const REPO_ROOT = process.cwd() +const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx') + +/** The fold's own accounting for the most recent open. */ +interface FoldReport { + bound: 'checkpoint' | 'low-water' | 'genesis' + fromGeneration: number + factsScanned: number + seeded: number + pending: number +} + +const roots: string[] = [] +const liveBrains: Brainy[] = [] + +function dir(): string { + const d = mkdtempSync(join(tmpdir(), 'brainy-embed-ckpt-')) + roots.push(d) + return d +} + +async function open(root: string, opts?: { blockWorker?: boolean }): Promise> { + const brain = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: root } + }) + // Blocking the worker BEFORE init() is how a "permanently stuck" pending id + // is built deterministically: the state under test is "an id the fold keeps + // re-arming and nothing ever disarms", and its production causes (a failing + // embedder, a wedged model, a data-less row) all reduce to exactly that. + if (opts?.blockWorker) (brain as unknown as { kickEmbedWorker: () => void }).kickEmbedWorker = () => {} + await brain.init() + liveBrains.push(brain) + return brain +} + +function foldReport(brain: Brainy): FoldReport { + const report = (brain as unknown as { _pendingEmbedFoldReport: FoldReport | null }) + ._pendingEmbedFoldReport + if (report === null) throw new Error('the open ran no pending-embed fold') + return report +} + +function pendingIds(brain: Brainy): string[] { + return [ + ...(brain as unknown as { _pendingEmbedIds: Set })._pendingEmbedIds + ].sort() +} + +/** Read an artifact straight off disk (the adapter gzips raw objects). */ +function readArtifact(root: string, path: string): Record | null { + const plain = join(root, ...path.split('/')) + const gz = `${plain}.gz` + if (existsSync(gz)) return JSON.parse(gunzipSync(readFileSync(gz)).toString('utf-8')) + if (existsSync(plain)) return JSON.parse(readFileSync(plain, 'utf-8')) + return null +} + +/** The on-disk path the adapter actually used for an artifact. */ +function artifactPath(root: string, path: string): string | null { + const plain = join(root, ...path.split('/')) + const gz = `${plain}.gz` + if (existsSync(gz)) return gz + if (existsSync(plain)) return plain + return null +} + +/** + * THE DIFFERENTIAL ORACLE: fold the log from generation 1 with exactly the + * engine's own rules. This is what the bounded fold must agree with, and its + * fact count is what the unbounded fold used to read at every open. + */ +async function fullFold(brain: Brainy): Promise<{ ids: string[]; facts: number }> { + const log = ( + brain as unknown as { generationStore: { getFactLog(): any } } + ).generationStore.getFactLog() + const pending = new Set() + let facts = 0 + const scan = log.scanFacts({ fromGeneration: 1 }) + for await (const batch of scan.batches()) { + for (const fact of batch.facts) { + facts++ + for (const record of fact.records ?? []) { + if (record.type === 'embed.pending') pending.add(record.id) + else if (record.type === 'embed.landed') pending.delete(record.id) + } + for (const op of fact.ops) { + if (op.kind === 'noun' && op.record === null) pending.delete(op.id) + } + } + } + return { ids: [...pending].sort(), facts } +} + +/** Capture every console.warn/error line emitted while `fn` runs. */ +async function captureConsole(fn: () => Promise): Promise<{ result: T; lines: string[] }> { + const lines: string[] = [] + const origWarn = console.warn + const origError = console.error + const sink = (...args: unknown[]) => { + lines.push(args.map((a) => String(a)).join(' ')) + } + console.warn = sink as typeof console.warn + console.error = sink as typeof console.error + try { + const result = await fn() + return { result, lines } + } finally { + console.warn = origWarn + console.error = origError + } +} + +/** + * Run a child process that arranges a store and then waits forever, so the + * parent can SIGKILL it. A real process death is the only honest way to pin + * "no close ran, no shutdown hook ran, RAM is gone". + * + * `detached` puts the child in its own process GROUP: tsx runs the script in a + * grandchild, and only a group-wide signal reaches the process holding the + * writer lock. + */ +function spawnArranger(root: string, body: string): Promise<{ + child: ReturnType + output: () => string +}> { + const scriptPath = join(root, 'arrange.mts') + writeFileSync(scriptPath, body) + const child = spawn(TSX, [scriptPath], { + cwd: REPO_ROOT, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true + }) + let out = '' + child.stdout!.on('data', (d) => { out += String(d) }) + child.stderr!.on('data', (d) => { out += String(d) }) + return new Promise((resolvePromise, rejectPromise) => { + const timer = setTimeout( + () => rejectPromise(new Error(`arranger never became READY:\n${out}`)), + 180_000 + ) + child.stdout!.on('data', () => { + if (out.includes('READY')) { + clearTimeout(timer) + resolvePromise({ child, output: () => out }) + } + }) + child.on('exit', (code) => { + clearTimeout(timer) + if (!out.includes('READY')) rejectPromise(new Error(`arranger exited ${code}:\n${out}`)) + }) + }) +} + +/** Parse the `IDS:{...}` line an arranger prints — supplied ids are normalised + * to canonical uuids, and the markers, checkpoint and fold all speak those. */ +function childIds(output: string): Record { + const line = output.split('\n').find((l) => l.startsWith('IDS:')) + if (!line) throw new Error(`arranger printed no IDS line:\n${output}`) + return JSON.parse(line.slice('IDS:'.length)) +} + +/** SIGKILL the whole group and wait for the grandchild's death to settle. */ +async function sigkill(child: ReturnType): Promise { + process.kill(-(child.pid as number), 'SIGKILL') + await new Promise((r) => child.on('exit', () => r())) + await new Promise((r) => setTimeout(r, 500)) +} + +/** The preamble every arranger child shares. */ +function childPreamble(root: string): string { + return ` + import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} + const ROOT = ${JSON.stringify(root)} + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ROOT } }) + const block = () => { (brain as any).kickEmbedWorker = () => {} } + const settleCheckpoint = async () => { + // The cadence write is fire-and-forget; wait for the single flight. + for (let i = 0; i < 200; i++) { + if (!(brain as any)._pendingEmbedCheckpointFlight) break + await (brain as any)._pendingEmbedCheckpointFlight.catch(() => {}) + } + } + ` +} + +afterEach(async () => { + for (const brain of liveBrains.splice(0)) { + try { await brain.close() } catch { /* already closed / crashed — teardown only */ } + } + for (const d of roots.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +// =========================================================================== +// 1. The stuck-id brain — the defect, and the bound that now engages on it +// =========================================================================== + +describe('pending-embed checkpoint — a brain whose pending set never drains', () => { + it('a permanently-stuck pending id: the reopen scans only the facts after the checkpoint', async () => { + const root = dir() + const first = await open(root, { blockWorker: true }) + // add() returns the CANONICAL id (supplied ids are normalised), and that is + // the id the markers, the checkpoint and the fold all speak. + const stuck = await first.add({ + id: 'stuck', + data: 'a deferred row whose embed never lands', + type: NounType.Thing, + deferEmbedding: true + }) + expect(first.pendingEmbedCount()).toBe(1) + // Ordinary traffic after it — every one of these is a fact the unbounded + // fold had to re-read at every open, forever, because of that one id. + for (let i = 0; i < 12; i++) { + await first.add({ id: `row-${i}`, data: `row ${i}`, type: NounType.Thing }) + } + await first.close() + liveBrains.splice(liveBrains.indexOf(first), 1) + + // THE DEFECT, PINNED: the pending set never drained, so the old bound was + // never written — nothing on this brain could have shortened its fold. + expect(readArtifact(root, LOWWATER_PATH)).toBeNull() + // The checkpoint IS written at the clean close, set non-empty and all. + const checkpoint = readArtifact(root, CHECKPOINT_PATH) as { + generation: number + pending: string[] + } | null + expect(checkpoint).not.toBeNull() + expect(checkpoint!.generation).toBeGreaterThan(0) + expect(checkpoint!.pending).toEqual([stuck]) + + const second = await open(root, { blockWorker: true }) + const report = foldReport(second) + // THE FIX, from the fold's own counter — not the clock. + expect(report.bound).toBe('checkpoint') + expect(report.fromGeneration).toBe(checkpoint!.generation + 1) + expect(report.factsScanned).toBe(0) + expect(report.seeded).toBe(1) + // The crash-recovery contract is intact: the marker is re-armed by open(). + expect(pendingIds(second)).toEqual([stuck]) + expect(second.pendingEmbedCount()).toBe(1) + + // The differential: the bounded answer is the full-fold answer, and the + // full fold is what the previous bound would have had to read. + const full = await fullFold(second) + expect(full.ids).toEqual([stuck]) + expect(full.facts).toBeGreaterThanOrEqual(13) + expect(report.factsScanned).toBeLessThan(full.facts) + }, 180_000) + + it('the bound stays O(delta) across repeated opens while the id is still stuck', async () => { + const root = dir() + const first = await open(root, { blockWorker: true }) + const stuck = await first.add({ + id: 'stuck', + data: 'never lands', + type: NounType.Thing, + deferEmbedding: true + }) + for (let i = 0; i < 6; i++) { + await first.add({ id: `a-${i}`, data: `a ${i}`, type: NounType.Thing }) + } + await first.close() + liveBrains.splice(liveBrains.indexOf(first), 1) + + const second = await open(root, { blockWorker: true }) + expect(foldReport(second).factsScanned).toBe(0) + // More history under the same stuck id. + for (let i = 0; i < 9; i++) { + await second.add({ id: `b-${i}`, data: `b ${i}`, type: NounType.Thing }) + } + await second.close() + liveBrains.splice(liveBrains.indexOf(second), 1) + + const third = await open(root, { blockWorker: true }) + const report = foldReport(third) + const full = await fullFold(third) + expect(report.bound).toBe('checkpoint') + expect(report.factsScanned).toBe(0) + // The unbounded fold grew with the store; the bounded one did not. + expect(full.facts).toBeGreaterThanOrEqual(16) + expect(pendingIds(third)).toEqual([stuck]) + expect(full.ids).toEqual([stuck]) + }, 180_000) +}) + +// =========================================================================== +// 2. Torn checkpoint — falls back, loudly, correctly +// =========================================================================== + +describe('pending-embed checkpoint — a torn checkpoint never shortens the fold', () => { + it('an undecodable checkpoint file degrades to the next bound, loudly, with the right pending set', async () => { + const root = dir() + const first = await open(root, { blockWorker: true }) + const stuck = await first.add({ + id: 'stuck', + data: 'never lands', + type: NounType.Thing, + deferEmbedding: true + }) + for (let i = 0; i < 5; i++) { + await first.add({ id: `row-${i}`, data: `row ${i}`, type: NounType.Thing }) + } + await first.close() + liveBrains.splice(liveBrains.indexOf(first), 1) + + const onDisk = artifactPath(root, CHECKPOINT_PATH) + expect(onDisk).not.toBeNull() + // Tear it: bytes that are neither valid gzip nor valid JSON. A torn file + // must THROW on read — never parse into a partial `pending` list. + writeFileSync(onDisk!, 'not a checkpoint at all {{{') + + const before = getTornRecordGauge().count + const { result: second, lines } = await captureConsole(async () => + open(root, { blockWorker: true }) + ) + const report = foldReport(second) + // Fell back — never to a shorter bound, and never silently. + expect(report.bound).not.toBe('checkpoint') + expect(report.seeded).toBe(0) + expect(report.fromGeneration).toBe(1) // no mark either: this brain never drained + // LOUD, two ways: the adapter's torn-record gauge and its production error… + expect(getTornRecordGauge().count).toBeGreaterThan(before) + expect(getTornRecordGauge().lastPath).toContain('pending_embeds_checkpoint') + expect(lines.some((l) => /TORN RECORD/.test(l))).toBe(true) + // …and the fold's own narration of which bound it actually used. + expect(lines.some((l) => /pending-embed fold: genesis bound/.test(l))).toBe(true) + + // CORRECT: the marker is still recovered, from the log itself. + expect(pendingIds(second)).toEqual([stuck]) + const full = await fullFold(second) + expect(full.ids).toEqual([stuck]) + expect(report.factsScanned).toBe(full.facts) + }, 180_000) + + it('a well-formed but shape-invalid checkpoint is refused whole, never partially trusted', async () => { + const root = dir() + const first = await open(root, { blockWorker: true }) + const stuck = await first.add({ + id: 'stuck', + data: 'never lands', + type: NounType.Thing, + deferEmbedding: true + }) + await first.add({ id: 'other', data: 'ordinary row', type: NounType.Thing }) + await first.close() + liveBrains.splice(liveBrains.indexOf(first), 1) + + // A checkpoint with a plausible generation but a `pending` that is not a + // list of ids: trusting the generation alone would bound the scan behind a + // set that was never recovered — the exact shape that loses a vector. + const onDisk = artifactPath(root, CHECKPOINT_PATH)! + const good = readArtifact(root, CHECKPOINT_PATH) as { generation: number } + rmSync(onDisk) + writeFileSync( + join(root, '_system', 'pending_embeds_checkpoint.json'), + JSON.stringify({ generation: good.generation, pending: { stuck: true }, writtenAt: 1 }) + ) + + const { result: second, lines } = await captureConsole(async () => + open(root, { blockWorker: true }) + ) + expect(lines.some((l) => /pending-embed checkpoint REFUSED/.test(l))).toBe(true) + const report = foldReport(second) + expect(report.bound).not.toBe('checkpoint') + expect(report.seeded).toBe(0) + expect(pendingIds(second)).toEqual([stuck]) + }, 180_000) +}) + +// =========================================================================== +// 3. The crash matrix — real processes, real SIGKILL, differential invariant +// =========================================================================== + +describe('pending-embed checkpoint — crash matrix (real child process, SIGKILL)', () => { + /** + * The invariant every row shares: whatever the reopened brain's fold did with + * whatever bound survived the crash, its pending set must equal the truth a + * full fold from generation 1 derives from the SAME recovered log. + */ + async function assertDifferentialAfterCrash(root: string): Promise<{ + report: FoldReport + full: { ids: string[]; facts: number } + pending: string[] + }> { + const reopened = await open(root, { blockWorker: true }) + const report = foldReport(reopened) + const full = await fullFold(reopened) + const pending = pendingIds(reopened) + expect(pending).toEqual(full.ids) + return { report, full, pending } + } + + it('killed BEFORE any checkpoint was written — falls back and recovers the marker from the log', async () => { + const root = dir() + const { child, output } = await spawnArranger( + root, + `${childPreamble(root)} + block() + await brain.init() + await brain.add({ id: 'landed-row', data: 'an ordinary row', type: 'thing' }) + const stuck = await brain.add({ id: 'stuck-1', data: 'deferred, never lands', type: 'thing', deferEmbedding: true }) + await brain.flush() + console.log('IDS:' + JSON.stringify({ stuck })) + console.log('READY') + setInterval(() => {}, 1000) + ` + ) + const ids = childIds(output()) + // One enqueue is well under the cadence and the set never drained, so no + // checkpoint exists — this is the pre-checkpoint crash. + expect(readArtifact(root, CHECKPOINT_PATH)).toBeNull() + await sigkill(child) + + const { report, pending } = await assertDifferentialAfterCrash(root) + expect(report.bound).toBe('genesis') + expect(pending).toEqual([ids.stuck]) + }, 300_000) + + it('killed AFTER a checkpoint, with an embed landed and flushed after it — the post-checkpoint facts carry the disarm', async () => { + const root = dir() + const { child, output } = await spawnArranger( + root, + `${childPreamble(root)} + await brain.init() + // Land one deferred embed: the drain arms the checkpoint debt. + await brain.add({ id: 'seed', data: 'lands first', type: 'thing', deferEmbedding: true }) + await brain.awaitPendingEmbeds() + await brain.flush() + // A second deferred write pays the debt (the head is at the manifest now), + // then LANDS — its embed.landed rides a fact ABOVE the checkpoint. + const landsAfter = await brain.add({ id: 'lands-after', data: 'lands after the checkpoint', type: 'thing', deferEmbedding: true }) + await settleCheckpoint() + await brain.awaitPendingEmbeds() + // …and one that never will. + block() + const stuck = await brain.add({ id: 'stuck-1', data: 'deferred, never lands', type: 'thing', deferEmbedding: true }) + await brain.add({ id: 'plain', data: 'more history', type: 'thing' }) + await brain.flush() + console.log('IDS:' + JSON.stringify({ stuck, landsAfter })) + console.log('READY') + setInterval(() => {}, 1000) + ` + ) + const ids = childIds(output()) + const checkpoint = readArtifact(root, CHECKPOINT_PATH) as { + generation: number + pending: string[] + } | null + expect(checkpoint).not.toBeNull() + await sigkill(child) + + const { report, full, pending } = await assertDifferentialAfterCrash(root) + expect(report.bound).toBe('checkpoint') + expect(report.fromGeneration).toBe(checkpoint!.generation + 1) + // The bound really bounded: fewer facts than the whole log. + expect(report.factsScanned).toBeLessThan(full.facts) + // A landed embed above the checkpoint is disarmed by the scan, not lost; + // the stuck one is re-armed. + expect(pending).toEqual([ids.stuck]) + expect(pending).not.toContain(ids.landsAfter) + }, 300_000) + + it('killed AFTER a checkpoint with an UN-FLUSHED tail — truncated facts and the bounded fold still agree', async () => { + const root = dir() + const { child } = await spawnArranger( + root, + `${childPreamble(root)} + await brain.init() + await brain.add({ id: 'seed', data: 'lands first', type: 'thing', deferEmbedding: true }) + await brain.awaitPendingEmbeds() + await brain.flush() + await brain.add({ id: 'lands-after', data: 'lands after the checkpoint', type: 'thing', deferEmbedding: true }) + await settleCheckpoint() + await brain.awaitPendingEmbeds() + await brain.flush() + // Now write PAST the manifest and never flush: these facts are the tail a + // crash truncates. Whatever survives, the two folds must agree on it. + block() + await brain.add({ id: 'stuck-tail', data: 'deferred, never lands', type: 'thing', deferEmbedding: true }) + await brain.add({ id: 'plain-tail', data: 'unflushed history', type: 'thing' }) + console.log('READY') + setInterval(() => {}, 1000) + ` + ) + const checkpoint = readArtifact(root, CHECKPOINT_PATH) as { generation: number } | null + expect(checkpoint).not.toBeNull() + await sigkill(child) + + const { report } = await assertDifferentialAfterCrash(root) + // The checkpoint's generation is at or below the manifest by construction, + // so it survived the truncation and still bounds the fold. + expect(report.bound).toBe('checkpoint') + expect(report.fromGeneration).toBe(checkpoint!.generation + 1) + }, 300_000) +}) From dee46b35c8bce51d1f581d710b55de403c2de803 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:17:05 -0700 Subject: [PATCH 100/150] ci(test): perf and scale benchmarks leave the correctness gate --- CONTRIBUTING.md | 14 ++++++++ package.json | 2 +- tests/configs/vitest.perf.config.ts | 56 +++++++++++++++++++++++++++++ vitest.config.ts | 33 +++++++++++++++-- 4 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 tests/configs/vitest.perf.config.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 54d4f784..c58520b7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,6 +41,20 @@ npm test Tests run on [Vitest](https://vitest.dev/). `npm test` runs the unit suite; see `package.json` for `test:integration`, `test:coverage`, and friends. +## Test gate + +The release gate is a bare `vitest run` (no `--config` flag) — the same +command the delta gate and CI's checks invoke. It carries the full +correctness suite and nothing else: wall-clock/scale benchmarks +(`tests/performance/**`, `tests/critical-performance-benchmark.test.ts`, +`tests/api/performance-benchmarks.test.ts`) and the two tests whose outcome +depends on the host machine or network rather than the code +(`tests/package-size-limit.test.ts` shells out to the `npm` CLI; +`tests/model-loading.test.ts` makes a real network call to download a model) +are excluded from it, because a timing threshold or a flaky network call has +no business failing a correctness check. That whole family runs on demand, +in its own exclusive slot, via `npm run test:perf`. + ## Standards - **Strict TypeScript.** No `any` escape hatches to dodge the type checker. diff --git a/package.json b/package.json index f07bb94c..f5a0325d 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "test:watch": "NODE_OPTIONS='--max-old-space-size=8192' vitest --config tests/configs/vitest.unit.config.ts", "test:coverage": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts --coverage", "test:unit": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.unit.config.ts", - "test:perf": "vitest run tests/unit/performance --reporter=basic", + "test:perf": "vitest run --config tests/configs/vitest.perf.config.ts", "test:integration": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.integration.config.ts", "test:semantic": "NODE_OPTIONS='--max-old-space-size=8192' vitest run --config tests/configs/vitest.semantic.config.ts", "test:all": "npm run test:unit && npm run test:integration", diff --git a/tests/configs/vitest.perf.config.ts b/tests/configs/vitest.perf.config.ts new file mode 100644 index 00000000..ca665dae --- /dev/null +++ b/tests/configs/vitest.perf.config.ts @@ -0,0 +1,56 @@ +import { defineConfig } from 'vitest/config' + +/** + * Perf/scale + environment-dependent test configuration. + * + * The exclusive on-demand slot for everything the correctness gate + * (`vitest.config.ts`, the config a bare `vitest run` picks up) excludes: + * wall-clock/scale benchmarks and the two tests whose outcome depends on + * the host machine or network rather than the code. See CONTRIBUTING.md's + * "Test gate" section and the exclude list in `vitest.config.ts` (root) for + * why each file lives here instead of the gate. + * + * `include` names this set explicitly — it is the mirror image of the + * root config's exclude list, not an independent glob, so the two stay in + * sync by inspection. Longer timeouts than the gate's 120s/60s: one case in + * tests/critical-performance-benchmark.test.ts measures ~128s of real work. + */ +export default defineConfig({ + test: { + globals: true, + setupFiles: ['./tests/setup.ts'], + environment: 'node', + + // Sequential, single fork — same isolation the gate uses, so a perf + // measurement isn't skewed by sibling test contention. + pool: 'forks', + poolOptions: { + forks: { + maxForks: 1, + minForks: 1, + singleFork: true, + isolate: true + } + }, + + testTimeout: 300000, // 5 minutes per test (the 128s case plus headroom) + hookTimeout: 120000, + teardownTimeout: 10000, + + maxConcurrency: 1, + fileParallelism: false, + + include: [ + 'tests/performance/**/*.{test,spec}.{js,ts}', + 'tests/critical-performance-benchmark.test.ts', + 'tests/api/performance-benchmarks.test.ts', + 'tests/package-size-limit.test.ts', + 'tests/model-loading.test.ts' + ], + + reporters: process.env.CI ? ['dot'] : ['basic'], + + retry: process.env.CI ? 1 : 0, + shard: process.env.VITEST_SHARD + } +}) diff --git a/vitest.config.ts b/vitest.config.ts index 116ab234..013c3c9b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,9 +2,16 @@ import { defineConfig } from 'vitest/config' /** * Vitest Configuration - Optimized for Memory-Intensive Tests - * + * * Handles ONNX transformer model testing (4-8GB memory requirement) * Based on 2024-2025 best practices + * + * THE CORRECTNESS GATE: this is the config a bare `vitest run` (no + * `--config` flag) picks up — the delta gate and CI both invoke it that + * way. See CONTRIBUTING.md's "Test gate" section for the full picture. + * Wall-clock/scale benchmarks and tests whose outcome depends on the host + * machine or network rather than the code are excluded below and run on + * demand instead, in their own slot: `npm run test:perf`. */ export default defineConfig({ test: { @@ -38,7 +45,29 @@ export default defineConfig({ 'node_modules/**', 'dist/**', 'scripts/**', - '**/*.browser.test.ts' + '**/*.browser.test.ts', + + // Wall-clock/scale benchmark family — timing assertions and scale + // sweeps whose pass/fail depends on the host machine's speed, not on + // the code. Whole files only (a file that mixes correctness describes + // with a perf describe stays in the gate). Run on demand via + // `npm run test:perf`, which targets exactly this list. + 'tests/performance/**', + 'tests/critical-performance-benchmark.test.ts', + 'tests/api/performance-benchmarks.test.ts', + + // Environment-dependent by construction, not timing-based: + // package-size-limit shells out to the `npm` CLI (not guaranteed + // present — the functional gate lane is Bun-only host-mode with no + // Node.js runtime) and parses npm-version-specific `npm pack` notice + // text; model-loading's "Real Model Download Integration" case makes + // a genuine, unmocked network call to HuggingFace (its own header + // says "Uses REAL transformer models - NO MOCKING"), and the whole + // file imports `../src/embeddings/model-manager.js`, which no longer + // exists anywhere under src/ — neither belongs in a gate that must be + // deterministic. + 'tests/package-size-limit.test.ts', + 'tests/model-loading.test.ts' ], // REPORTERS: Dot for CI, verbose for local From 65493ba2de09e291972b0539bf49c221e3f18ce7 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:34:13 -0700 Subject: [PATCH 101/150] fix(vfs): a path-scoped search is a served range over the path, not a refused prefix match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `vfs.search({ path })` built its scope as `path: { $startsWith: path }`. `$startsWith` is not in the filter vocabulary at all, and the `$`-less spelling is REFUSED by the metadata index's served-operator law — an equality/range posting index cannot evaluate a substring without reading every row, so it refuses rather than answering an empty page. Every path-scoped VFS search threw on this engine line; the two pins in tests/vfs/vfs.unit.test.ts that exercise it have been red since the operator law landed. The scope is now a half-open range over `metadata.path`: `[dir + '/', dir + '0')`. Every descendant path begins with `dir + '/'`, and '0' is the code point directly after '/', so membership in the range is EXACTLY "carries that prefix" — and because the bounds differ at one ASCII position the answer is identical under code-unit and code-point collation. Siblings fall out correctly for the same reason: `/scope-sibling/x` sorts below the lower bound and `/scope0` sits at the open upper bound. `recursive: false` narrows to the directory's own identity instead — `parent`, an indexed equality. The root adds no clause, because every VFS entity is under it. `path` is the VFS's truth (write and rename maintain it; the `Contains` edges are a projection of it), it is already indexed on every VFS entity, and `explain()` reports the range as `column-store` — "O(log n) binary search + roaring bitmap". So the scope narrows the search before it runs: no tree walk, no migration, no backfill, and nothing fetched that the scope then discards. Two other shapes were considered and rejected. A graph-scoped walk over `Contains` reads the projection rather than the truth and costs O(subtree) adjacency lookups per search, with the subtree's height as an unknown `depth`. An indexed `ancestors: string[]` field cannot be implemented honestly today: the index extractor skips arrays longer than ten elements, so a path more than ten levels deep would silently drop out of every scoped search — and it needs a backfill besides. Pinned in tests/vfs/vfs-search-path-scope.test.ts (all eight red before this change): descendants at three depths and never a sibling, including the `/scope-sibling` and `/scope0` prefix traps; a trailing or doubled slash names the same scope; the root scope equals the unscoped search; `recursive: false` is the immediate children and refuses a missing directory by name; every operator the search emits is ANSWERED by the index's own door rather than refused; the id universe the index resolves for the search is already the scope; and the range agrees with walking the tree. --- src/vfs/VirtualFileSystem.ts | 73 ++++++++++- tests/vfs/vfs-search-path-scope.test.ts | 165 ++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 tests/vfs/vfs-search-path-scope.test.ts diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 1a4b9fa5..bccd6fea 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -1572,7 +1572,19 @@ export class VirtualFileSystem implements IVirtualFileSystem { // ============= Semantic Operations ============= /** - * Search files with natural language + * Search files with natural language. + * + * `options.path` scopes the search to a directory: its whole subtree by + * default, its immediate children when `recursive` is `false`. Both scopes + * are metadata filters the index SERVES, so the scope narrows the search + * before it runs — no tree walk, and never an over-fetch filtered afterwards. + * + * @param query - The natural-language query. + * @param options - Scope, metadata filters and paging (see {@link SearchOptions}). + * @returns The matching files, best first. + * @throws {VFSError} ENOENT when `recursive: false` names a path that does + * not exist (the non-recursive scope is the directory's own identity, so + * the directory has to be there). */ async search(query: string, options?: SearchOptions): Promise { await this.ensureInitialized() @@ -1588,11 +1600,26 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } - // Add path filter if specified + // Scope to a directory, if asked. This used to emit + // `path: { $startsWith }` — an operator that is not in the filter + // vocabulary at all, and whose `$`-less spelling the metadata index + // REFUSES by the served-operator law (an equality/range posting index + // cannot evaluate a substring without reading every row). Every + // path-scoped VFS search therefore threw, and none has ever worked on + // this engine line. Both scopes below are served shapes. if (options?.path) { - params.where = { - ...params.where, - path: { $startsWith: options.path } + if (options.recursive === false) { + // Immediate children only: the directory's identity IS the scope, and + // `parent` is an indexed equality on every VFS entity. + params.where = { + ...params.where, + parent: await this.pathResolver.resolve(options.path) + } + } else { + const scope = this.descendantPathScope(options.path) + if (scope) { + params.where = { ...params.where, path: scope } + } } } @@ -1754,6 +1781,42 @@ export class VirtualFileSystem implements IVirtualFileSystem { return entity as VFSEntity } + /** + * The SERVED metadata shape for "everything under this directory". + * + * `metadata.path` is the VFS's truth — write and rename maintain it, and the + * `Contains` edges are a projection of it (see {@link repairContainment}) — + * it is indexed on every VFS entity, and the metadata index serves ordered + * range operators. So a subtree scope is a half-open range over the path + * column: O(log n + matches), no tree walk, and nothing fetched that the + * scope then discards. + * + * The range is `[dir + '/', dir + )`. Every descendant path + * begins with `dir + '/'`, and '0' is the code point directly after '/', so a + * string lies in the range EXACTLY when it carries that prefix. The two + * bounds differ at a single ASCII position, so the answer is the same under + * code-unit and code-point collation alike — no dependence on how the store + * orders the rest of the string. + * + * Sibling exclusion falls out of the same fact and is worth stating, because + * it is where a naive prefix test goes wrong: for `dir = '/scope'`, + * `/scope-sibling/x` sorts BELOW the lower bound ('-' precedes '/') and + * `/scope0` sits at the open upper bound — both outside, while + * `/scope/sub/deep/c.txt` is inside at any depth. + * + * @param path - The directory to scope to. + * @returns The `where` fragment for the `path` field, or `null` for the root + * — every VFS entity is under it, so no clause narrows the search. + */ + private descendantPathScope(path: string): { gte: string; lt: string } | null { + const dir = path.replace(/\/+/g, '/').replace(/\/$/, '') || '/' + if (dir === '/') return null + // Computed, so the bound carries its own reason: the first string that can + // no longer share the `dir + '/'` prefix. + const separatorSuccessor = String.fromCharCode('/'.charCodeAt(0) + 1) + return { gte: `${dir}/`, lt: `${dir}${separatorSuccessor}` } + } + private getParentPath(path: string): string { const normalized = path.replace(/\/+/g, '/').replace(/\/$/, '') const lastSlash = normalized.lastIndexOf('/') diff --git a/tests/vfs/vfs-search-path-scope.test.ts b/tests/vfs/vfs-search-path-scope.test.ts new file mode 100644 index 00000000..fd5fa4d5 --- /dev/null +++ b/tests/vfs/vfs-search-path-scope.test.ts @@ -0,0 +1,165 @@ +/** + * @module tests/vfs/vfs-search-path-scope + * @description `vfs.search({ path })` scopes with a SERVED filter. + * + * The scope used to be emitted as `path: { $startsWith }` — an operator that is + * not in the filter vocabulary at all, and whose `$`-less spelling the metadata + * index refuses by the served-operator law (an equality/range posting index + * cannot evaluate a substring without reading every row). Every path-scoped VFS + * search threw; none has ever worked on this engine line. + * + * The scope is now a half-open range over `metadata.path`, which is the VFS's + * truth, is indexed on every VFS entity, and is served by the ordered range + * operators: `[dir + '/', dir + '0')` — '0' being the code point after '/', so + * membership in the range is EXACTLY "carries the prefix `dir/`". The + * non-recursive scope is the directory's own identity, `parent`, an equality. + * + * These pins hold the answer (descendants at every depth, siblings never — the + * `/scope-sibling` trap included), the shape (the operators the search emits + * are answered by the index's own door, never refused), and the law that the + * scope narrows the search BEFORE it runs rather than filtering an over-fetch. + */ +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' +import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' +import { Brainy } from '../../src/brainy.js' +import { VFSErrorCode } from '../../src/vfs/types.js' + +/** A word every fixture file carries, so the text leg reaches all of them. */ +const TOKEN = 'quasar' + +describe('vfs.search({ path }) scopes with a served filter', () => { + let brain: Brainy + let vfs: VirtualFileSystem + + /** In scope for '/scope', at three depths. */ + const inScope = ['/scope/a.txt', '/scope/sub/b.txt', '/scope/sub/deep/c.txt'] + /** Out of scope — including the two prefix traps a naive test misses. */ + const outOfScope = ['/scope-sibling/d.txt', '/scope0/e.txt', '/elsewhere/f.txt', '/g.txt'] + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) + await brain.init() + vfs = brain.vfs + await vfs.init() + + await vfs.mkdir('/scope/sub/deep', { recursive: true }) + await vfs.mkdir('/scope-sibling', { recursive: true }) + await vfs.mkdir('/scope0', { recursive: true }) + await vfs.mkdir('/elsewhere', { recursive: true }) + + for (const path of [...inScope, ...outOfScope]) { + await vfs.writeFile(path, `${TOKEN} content for ${path}`) + } + }) + + afterAll(async () => { + await vfs?.close() + await brain?.close() + }) + + it('includes every descendant depth and excludes every sibling', async () => { + const results = await vfs.search(TOKEN, { path: '/scope', limit: 50 }) + const paths = results.map((r) => r.path).sort() + + expect(paths).toEqual([...inScope].sort()) + for (const path of outOfScope) expect(paths).not.toContain(path) + }) + + it('a trailing slash and a doubled slash name the same scope', async () => { + const plain = await vfs.search(TOKEN, { path: '/scope', limit: 50 }) + const trailing = await vfs.search(TOKEN, { path: '/scope/', limit: 50 }) + const doubled = await vfs.search(TOKEN, { path: '//scope//', limit: 50 }) + + const ids = (rs: Array<{ entityId: string }>) => rs.map((r) => r.entityId).sort() + expect(ids(trailing)).toEqual(ids(plain)) + expect(ids(doubled)).toEqual(ids(plain)) + }) + + it('the root scope is every VFS file — it adds no clause to narrow with', async () => { + const rooted = await vfs.search(TOKEN, { path: '/', limit: 50 }) + const unscoped = await vfs.search(TOKEN, { limit: 50 }) + + const paths = rooted.map((r) => r.path).sort() + expect(paths).toEqual([...inScope, ...outOfScope].sort()) + expect(paths).toEqual(unscoped.map((r) => r.path).sort()) + }) + + it('recursive: false is the immediate children, not the subtree', async () => { + const results = await vfs.search(TOKEN, { path: '/scope', recursive: false, limit: 50 }) + expect(results.map((r) => r.path)).toEqual(['/scope/a.txt']) + }) + + it('recursive: false on a path that does not exist refuses by name', async () => { + await expect( + vfs.search(TOKEN, { path: '/no-such-dir', recursive: false, limit: 50 }) + ).rejects.toMatchObject({ code: VFSErrorCode.ENOENT }) + }) + + it('every operator the search emits is ANSWERED by the index door, never refused', async () => { + const index = (brain as any).metadataIndex + const emitted: any[] = [] + const find = vi.spyOn(brain as any, 'find') + try { + await vfs.search(TOKEN, { path: '/scope', limit: 50 }) + await vfs.search(TOKEN, { path: '/scope/sub', where: { mimeType: 'text/plain' }, limit: 50 }) + await vfs.search(TOKEN, { path: '/scope', recursive: false, limit: 50 }) + await vfs.search(TOKEN, { path: '/', limit: 50 }) + for (const call of find.mock.calls) emitted.push((call[0] as any).where) + } finally { + find.mockRestore() + } + + expect(emitted).toHaveLength(4) + for (const where of emitted) { + // The door itself is the judge: an operator outside the served set is + // REFUSED here (BrainyError INVALID_QUERY), never answered. + await expect(index.getIdsForFilter(where)).resolves.toBeInstanceOf(Array) + } + + // And the scope really is a range on the path — the shape this fix chose. + expect(emitted[0].path).toEqual({ gte: '/scope/', lt: '/scope0' }) + expect(emitted[3].path).toBeUndefined() + }) + + it('the scope narrows the search before it runs — no over-fetch to filter', async () => { + const index = (brain as any).metadataIndex + const filter = vi.spyOn(index, 'getIdsForFilter') + let universe: string[] = [] + try { + await vfs.search(TOKEN, { path: '/scope', limit: 50 }) + // The search's own call — the one carrying the scope. (Path resolution + // asks this same door for the root, before the search is built.) + const scoped = filter.mock.calls.findIndex( + (c) => (c[0] as any)?.path?.gte === '/scope/' + ) + expect(scoped).toBeGreaterThanOrEqual(0) + universe = (await filter.mock.results[scoped].value) as string[] + } finally { + filter.mockRestore() + } + + // The id universe the index resolved for the search is already the scope: + // three files, and not one row from outside it. + const rows = await brain.batchGet(universe) + const paths = [...rows.values()].map((e: any) => e.metadata.path).sort() + expect(paths).toEqual([...inScope].sort()) + }) + + it('the range answers the same ids as walking the tree', async () => { + // The path is the truth and the Contains edges are its projection; a scope + // read from the truth must agree with one walked over the projection. + const walked: string[] = [] + const walk = async (dir: string): Promise => { + for (const name of await vfs.readdir(dir)) { + const child = dir === '/' ? `/${name}` : `${dir}/${name}` + const stat = await vfs.stat(child) + if (stat.isDirectory()) await walk(child) + else walked.push(child) + } + } + await walk('/scope') + + const searched = await vfs.search(TOKEN, { path: '/scope', limit: 50 }) + expect(searched.map((r) => r.path).sort()).toEqual(walked.sort()) + }) +}) From ec644bde56ec6a052f400b776e25dc58691135be Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:41:55 -0700 Subject: [PATCH 102/150] =?UTF-8?q?fix(shutdown):=20one=20owner=20per=20br?= =?UTF-8?q?ain=20=E2=80=94=20the=20signal=20handler=20defers=20to=20close(?= =?UTF-8?q?),=20and=20flush=20is=20single-flight?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEASURED IN PRODUCTION. A host that owns its own shutdown — one SIGTERM listener calling close() on every pooled store — ran head-on into the engine's own signal handler, which iterated every live instance, flushed its components in parallel, and released its writer lock in its own finally. Two teardowns of the same brain at the same moment: "Shutdown signal received - flushing pending data...", 148s of silence, "Flushed successfully (1 instance)", and the host's pool close of that same store returning 1s later — 149s against 24s for the six stores with no engine work in flight. The same race reproduced locally as "Failed to flush one Brainy instance on shutdown: Writer fence lost … the lock file is gone": the handler observing a lock the close it was racing had already released. Three changes, one law — a brain's teardown belongs to whoever started it. 1. close() is idempotent and re-entrant. The first call stores its promise synchronously in _closeInFlight and every later or concurrent caller gets that same promise back; the teardown runs once. close() is no longer async so the promise is shared by identity, not just outcome. The state is observable: isClosing (begun) and isClosed (finished). 2. The signal handler defers one macrotask, then per instance either steps aside (a close has begun or finished — its owner owns the flush, the markers and the lock) or awaits instance.close(): the same settle/flush/attest/ marker/lock path any caller gets. Its old parallel per-component flush and separate lock release are gone; the three laws that block carried are each satisfied by close(), verified line by line and recorded in the new comment. Per-instance isolation stays here, in the loop's try/catch. Sole-owner exit now reads the listener count WHEN THE SIGNAL ARRIVES. Asking afterwards reads a process that has already torn itself down — closing the last brain deregisters the engine's own listeners, so a host's single remaining listener would look like "<= 1" and be force-exited out of its own graceful shutdown. 3. Flush is single-flight with a queue one deep. It did not coalesce: the cadence's guard covered only the flushes the cadence started, so a cross-process flush request or an application flush() overlapped it freely — production showed two "Flushing Brainy indexes…" runs 3s apart, walls growing 295ms to 4.9s. The gate now lives in flush() itself and covers every caller: run, or join the ONE queued follow-up. A follow-up rather than joining the running flush, because a caller flushes to make ITS writes durable and those may have landed after the running flush read its state; it costs nothing when there is nothing new. close() drains that chain too. The idle law is untouched: a clean brain's flush still returns immediately, and an idle brain still flushes zero times. --- src/brainy.ts | 364 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 267 insertions(+), 97 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 7568a6f3..39b604ad 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -767,6 +767,34 @@ export class Brainy implements BrainyInterface { private _persistIdleTimer: ReturnType | null = null private _persistBackgroundFlight: Promise | null = null + /** + * FLUSH IS SINGLE-FLIGHT, AND THE QUEUE IS ONE DEEP. `_flushInFlight` is the + * flush body actually running; `_flushFollowUp` is the AT MOST ONE flush + * queued behind it. Every caller — the write cadence, the cross-process + * flush-request watcher, an application calling `flush()` directly — either + * runs (nothing in flight), or joins the single queued follow-up. + * + * WHY A FOLLOW-UP RATHER THAN JOINING THE RUNNING FLUSH: a caller flushes to + * make ITS writes durable, and those writes may have landed after the + * running flush read its state. Joining would return "flushed" over data + * that was never persisted. Chaining one follow-up costs nothing when there + * is nothing new (a clean brain's flush returns immediately — see + * `_dirtySinceLastFlush`) and is correct when there is. + * + * MEASURED, in the production shutdown this was written for: two + * "Flushing Brainy indexes and caches to disk..." runs overlapping 3s + * apart on one brain, their walls growing 295ms → 4.9s as they contended + * for the same providers. + */ + private _flushInFlight: Promise | null = null + private _flushFollowUp: Promise | null = null + /** Flush bodies that got past the single-flight gate (pinned by tests). */ + private _flushBodyRuns = 0 + /** Flush bodies running right now, and the high-water mark — which the + * single-flight law requires to stay at 1 (pinned by tests). */ + private _flushBodiesActive = 0 + private _flushConcurrencyPeak = 0 + // DEFERRED EMBEDDING (MT5): pending markers are LOG RECORDS — an // embed.pending record rides the deferred write's own commit fact and // embed.landed rides the landing commit; this set is the in-memory @@ -889,6 +917,24 @@ export class Brainy implements BrainyInterface { // applies only to instances that were never closed. private closed = false + /** + * THE ONE CLOSE. Set SYNCHRONOUSLY by the first `close()` call, before that + * call yields, and never cleared — close is terminal. Every later or + * concurrent caller receives this same promise, so a shutdown with two + * callers (a host's pool close and the engine's own signal handler) runs + * ONE teardown, not two. + * + * MEASURED, the day this was added: a host that owns shutdown called + * `close()` on every pooled store at SIGTERM while the engine's signal + * handler flushed the same instances in parallel and released their writer + * locks in its own `finally`. One store took 149s to close (148s of it + * silent) against 24s for its idle siblings, and the same race in a local + * reproduction printed `Writer fence lost … the lock file is gone` — the + * handler observing a lock the close it was racing had already released. + * Two owners of one shutdown; now there is one, whoever calls first. + */ + private _closeInFlight: Promise | null = null + // Index-build-at-open state. `lazyRebuildCompleted` predates the health-gate // law (it named a first-QUERY lazy rebuild) and stays for `getIndexStatus()` // API compatibility, but its truth changed: a needed rebuild now runs @@ -2076,105 +2122,88 @@ export class Brainy implements BrainyInterface { */ private registerShutdownHooks(): void { /** - * The signal-path shutdown. THREE LAWS, each written by a production - * shutdown that looked clean and wasn't: + * The signal-path shutdown. ONE OWNER PER BRAIN, AND THE PATH IS `close()`. * - * 1. PER-INSTANCE ISOLATION. This used to be one `try` around a loop over - * every open brain: the first instance whose flush rejected aborted the - * loop, so every remaining brain kept its writer lock and its unwritten - * markers — and the process still exited 0. A pool of brains failed in - * a batch, not one at a time. - * 2. THE MARKER IS PART OF SHUTDOWN. Flushing the indexes without closing - * the generation store leaves the clean-shutdown marker unwritten, so - * the NEXT open reads the store as crashed and folds the whole - * generation log — measured in tens of seconds on a real store, paid on - * every restart, after a shutdown the operator saw exit 0. - * 3. THE LOCK IS ALWAYS GIVEN UP. In a `finally`, per instance: a process - * on its way out holds nothing. + * WHAT THIS REPLACED, and why. The handler used to run its own shutdown — + * a parallel per-component flush, the generation store's close, a second + * parallel round of component closes, and a `finally` that stopped the + * flush-request watcher and released the writer lock. That is a SECOND + * teardown of the same brain, and a host application with its own SIGTERM + * handler (the shape every pooled deployment has) ran the FIRST one at the + * same moment. MEASURED in production the day this changed: a host closing + * seven pooled stores at SIGTERM printed "Shutdown signal received - + * flushing pending data...", went silent for 148s, printed "Flushed + * successfully (1 instance)", and the host's own close of that same store + * returned 1s later — 149s, against 24s for the six stores with no engine + * work in flight. The same race reproduced locally as + * `Failed to flush one Brainy instance on shutdown: Writer fence lost … + * the lock file is gone`: this handler observing a lock that the close it + * was racing had already released. + * + * SO: defer one macrotask, then per instance either STEP ASIDE (a close + * has begun or finished — its owner owns the flush, the markers and the + * lock) or `await instance.close()` — the one durable path, identical to + * what any caller gets. The three laws the old block carried are all + * satisfied by `close()`, each verified against its code: + * + * 1. PER-INSTANCE ISOLATION — kept HERE, in the per-instance try/catch + * below: one brain's failed close never aborts the loop over the rest. + * (`close()` itself is per-instance by construction.) + * 2. THE MARKER IS PART OF SHUTDOWN — `close()` → `closeDurableSteps()` + * Phase 1 awaits `this.generationStore.close()`, which persists the + * counter, advances the fold checkpoint and stamps the clean-shutdown + * marker LAST. That is the step that decides adopt-vs-fold at the next + * open, and it is the same call the old block made. + * 3. THE LOCK IS ALWAYS GIVEN UP — `close()`'s terminal releases run + * whether the durable steps threw or not (its contract: "TWO PARTS, AND + * THE SECOND IS UNCONDITIONAL"): `stopFlushRequestWatcher()` then + * `releaseWriterLock()`, then the VFS shutdown and the terminal + * `closed` flag, and only then is the original failure rethrown. + * `close()` releases the lock in MORE cases than the old block did — it + * also drains the metadata write buffer first, so no pending write can + * land after a successor writer claims the lock. */ - const flushOnShutdown = async () => { + const closeOnShutdown = async () => { console.log('Shutdown signal received - flushing pending data...') - let flushedCount = 0 + // DEFER ONE MACROTASK. A host application registers its own listener on + // the same signal, and Node runs listeners in registration order — ours + // is usually first, because the brain was opened before the host wired + // its shutdown. Yielding once lets every other listener for this signal + // run its synchronous prologue, so a host that calls close() gets to be + // the owner. It is only a courtesy, never the safety: close()'s own + // single-flight gate is what makes a lost race harmless. + await new Promise((resolve) => setImmediate(resolve)) + + let closedCount = 0 + let deferredCount = 0 let failedCount = 0 // Snapshot: close() splices Brainy.instances while we iterate. for (const instance of [...Brainy.instances]) { if (!instance.initialized) continue + // SOMEONE ELSE OWNS THIS ONE. Not a flush, not a lock release, not a + // component close — nothing. Touching a brain whose close is running + // is the whole defect this handler was rewritten for. + if (instance.closed || instance._closeInFlight !== null) { + deferredCount++ + continue + } try { - // Flush all buffered data (parallel across components, this brain only). - await Promise.all([ - (async () => { - if (instance.storage && typeof instance.storage.flushCounts === 'function') { - await instance.storage.flushCounts() - } - })(), - (async () => { - if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { - await instance.metadataIndex.flush() - } - })(), - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { - await instance.graphIndex.flush() - } - })(), - (async () => { - if (instance.index && typeof instance.index.flush === 'function') { - await instance.index.flush() - } - })() - ]) - - // Close the generation store: persists the counter, advances the - // fold checkpoint, and stamps the clean-shutdown marker LAST — the - // one step that decides whether the next open adopts or folds. Law 2. - if (instance.generationStore && !instance.isReadOnly) { - await instance.generationStore.close() - } - - // Close components to stop timers that would prevent clean process exit - await Promise.all([ - (async () => { - if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { - await instance.graphIndex.close() - } - })(), - (async () => { - const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks - if (index && typeof index.close === 'function') { - await index.close() - } - })(), - (async () => { - const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks - if (metadataIndex && typeof metadataIndex.close === 'function') { - await metadataIndex.close() - } - })() - ]) - flushedCount++ + // Law 1: this try/catch is the isolation — the loop continues. + await instance.close() + closedCount++ } catch (error) { failedCount++ - console.error('Failed to flush one Brainy instance on shutdown:', error) - } finally { - // Law 3 — the lock and the watcher go regardless. - try { - if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { - instance.storage.stopFlushRequestWatcher() - } - } catch (error) { - console.error('Failed to stop the flush-request watcher on shutdown:', error) - } - try { - if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { - await instance.storage.releaseWriterLock() - } - } catch (error) { - console.error('Failed to release the writer lock on shutdown:', error) - } + console.error('Failed to close one Brainy instance on shutdown:', error) } } - if (flushedCount > 0) { - console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) + if (closedCount > 0) { + console.log(`Flushed successfully (${closedCount} instance${closedCount > 1 ? 's' : ''})`) + } + if (deferredCount > 0) { + console.log( + `${deferredCount} Brainy instance${deferredCount > 1 ? 's are' : ' is'} already ` + + `closing — left to the caller that owns that close.` + ) } if (failedCount > 0) { console.error( @@ -2201,19 +2230,29 @@ export class Brainy implements BrainyInterface { * markers unwritten. When the host has its own handler (listener count * above our own), the host owns the exit; Brainy only makes its data * durable and steps aside. + * + * THE COUNT IS TAKEN WHEN THE SIGNAL ARRIVES, not after the shutdown ran. + * "Is anyone else handling this signal?" is a question about the moment + * the signal landed. Asking afterwards reads a process that has already + * torn itself down: the handler now CLOSES its instances, and closing the + * last brain deregisters Brainy's own listeners — so a host application's + * single remaining listener would look like `<= 1` and get force-exited + * out of its own graceful shutdown, precisely the failure above. */ - const exitIfSoleShutdownOwner = (signal: 'SIGTERM' | 'SIGINT'): void => { - if (process.listenerCount(signal) <= 1) { + const exitIfSoleShutdownOwner = (ownersWhenSignalled: number): void => { + if (ownersWhenSignalled <= 1) { process.exit(0) } } Brainy.sigtermListener = async () => { - await flushOnShutdown() - exitIfSoleShutdownOwner('SIGTERM') + const owners = process.listenerCount('SIGTERM') + await closeOnShutdown() + exitIfSoleShutdownOwner(owners) } Brainy.sigintListener = async () => { - await flushOnShutdown() - exitIfSoleShutdownOwner('SIGINT') + const owners = process.listenerCount('SIGINT') + await closeOnShutdown() + exitIfSoleShutdownOwner(owners) } Brainy.beforeExitListener = async () => { // Self-deregister FIRST: Node re-emits 'beforeExit' after every event- @@ -2225,7 +2264,7 @@ export class Brainy implements BrainyInterface { process.off('beforeExit', Brainy.beforeExitListener) Brainy.beforeExitListener = undefined } - await flushOnShutdown() + await closeOnShutdown() } process.on('SIGTERM', Brainy.sigtermListener) process.on('SIGINT', Brainy.sigintListener) @@ -2298,6 +2337,33 @@ export class Brainy implements BrainyInterface { return this.initialized } + /** + * @description Whether `close()` has BEGUN on this instance — in flight or + * already finished. The question a shutdown owner asks: this brain's + * teardown belongs to whoever started it, and a second party must not flush + * its components or release its writer lock underneath it. + * + * True from the synchronous moment `close()` is entered, so a listener that + * yields a tick and comes back reads the truth, not a stale "not yet". + * @returns `true` once a close has started. + */ + get isClosing(): boolean { + return this._closeInFlight !== null + } + + /** + * @description Whether `close()` has FINISHED tearing this instance down — + * durable steps attempted, writer lock released, instance terminal. A + * closed brain never re-initializes; every operation on it throws. + * + * True after a close that FAILED partway, too: such a brain still holds no + * writer lock and still serves nothing (see {@link close}). + * @returns `true` once the teardown has completed. + */ + get isClosed(): boolean { + return this.closed + } + /** * Promise that resolves when Brainy is fully initialized and ready to use * @@ -3271,9 +3337,18 @@ export class Brainy implements BrainyInterface { * toward the next trigger. A failure is LOUD and leaves the writes counted * again — silence is not an option, and neither is a retry storm (the next * trigger re-attempts). + * + * COALESCING LIVES IN {@link flush}, NOT HERE. A kick that arrives while a + * flush is running used to return without doing anything — the writes it + * counted waited for some LATER trigger, and this method's guard also could + * not coalesce the flushes it does not start (the cross-process + * flush-request watcher and application `flush()` calls both go straight to + * `flush()`; two of those overlapping is exactly what production showed). + * The gate in `flush()` covers every caller: this kick now either runs the + * flush or joins the single queued follow-up, so the writes it counted are + * always someone's work, and there is still never a second concurrent run. */ private kickBackgroundFlush(reason: 'threshold' | 'idle'): void { - if (this._persistBackgroundFlight) return const counted = this._persistDirtyWrites this._persistDirtyWrites = 0 this._persistLastFlushAt = Date.now() @@ -12903,7 +12978,58 @@ export class Brainy implements BrainyInterface { * process.exit(0) * }) */ - async flush(): Promise { + flush(): Promise { + // ---- THE SINGLE-FLIGHT GATE ---- + // One flush body runs at a time, with at most ONE queued behind it. See + // `_flushInFlight` / `_flushFollowUp` for the measurement that required + // this. NOT `async`: the gate hands back the very promise the work is on, + // so joining callers share identity, not just an outcome. The gate is + // crossed BEFORE any await, so two callers in the same tick cannot both + // find the field empty. + if (this._flushInFlight) { + if (!this._flushFollowUp) { + // The running flush's failure is not this follow-up's failure: it is + // reported to ITS caller, and the queued work still gets its turn. + this._flushFollowUp = this._flushInFlight + .catch(() => {}) + .then(() => { + this._flushFollowUp = null + return this.flush() + }) + } + return this._flushFollowUp + } + const run = this._runFlush() + // `finally` and not `then`: a failed flush must still open the gate, or + // one rejection would wedge every later flush behind a promise nobody + // will ever settle. + const gated = run.finally(() => { + if (this._flushInFlight === gated) this._flushInFlight = null + }) + this._flushInFlight = gated + return gated + } + + /** + * @description The flush body — everything {@link flush} promises, run + * exactly once at a time by that method's single-flight gate. Private + * because non-overlap is part of the contract: there is no supported way to + * run two of these at once, and the counters here witness that. + * @returns Nothing. + */ + private async _runFlush(): Promise { + this._flushBodyRuns++ + this._flushBodiesActive++ + this._flushConcurrencyPeak = Math.max(this._flushConcurrencyPeak, this._flushBodiesActive) + try { + await this._flushSteps() + } finally { + this._flushBodiesActive-- + } + } + + /** @description The flush steps themselves. See {@link flush}. */ + private async _flushSteps(): Promise { await this.ensureInitialized() // Read-only instances have no buffered writes to flush. close() may call @@ -20150,11 +20276,42 @@ export class Brainy implements BrainyInterface { * * The original failure is never swallowed: it is narrated with what it costs * the next open, then rethrown to the caller. + * + * IDEMPOTENT AND RE-ENTRANT. The teardown below runs ONCE. Concurrent + * callers share the one in-flight promise and settle together; a caller + * arriving after it finished gets that same settled promise (close is + * terminal — there is nothing left to redo, and a failed close has already + * released the lock and set `closed`). This is what makes the shutdown + * ownership question answerable at all: whoever calls first owns the close, + * everyone else — including the engine's own signal handler — joins it or + * steps aside. See `_closeInFlight`. * @returns Nothing. * @throws The first failure from the durable close steps, after the * terminal releases have run. */ - async close(): Promise { + close(): Promise { + // NOT `async`: an async wrapper allocates a FRESH promise per call, so + // callers would hold different handles to the same work. Returning the + // stored promise itself makes "one close" observable identity, not just + // observable behaviour. The gate is crossed with NO await before it, so + // two callers in the same tick — and a signal handler resuming mid-close + // — always see the same answer; `isClosing` is true from this assignment + // onward. (`_closeOnce()` is async, so a failure is always a rejection, + // never a synchronous throw out of this method.) + if (this._closeInFlight) return this._closeInFlight + const run = this._closeOnce() + this._closeInFlight = run + return run + } + + /** + * @description The close body — everything {@link close} promises, run + * exactly once by that method's gate. + * @returns Nothing. + * @throws The first failure from the durable close steps, after the + * terminal releases have run. + */ + private async _closeOnce(): Promise { if (this._pendingEmbedIds.size === 0) await this.writeEmbedLowWater() let closeFailure: unknown = null try { @@ -20243,6 +20400,19 @@ export class Brainy implements BrainyInterface { if (this._persistBackgroundFlight) { await this._persistBackgroundFlight.catch(() => {}) } + // Drain the flush chain itself: the running flush AND the single follow-up + // queued behind it. The cadence's own handle above covers only the flushes + // the cadence started — a flush-request from another process, or an + // application's own flush() racing this close, is on the chain and nowhere + // else, and a flush landing mid-close writes behind the close's work. + // Bounded by construction: at most one follow-up exists, and awaiting it + // awaits its leader too, so the second pass is a no-op unless a writer + // raced this close. + for (let pass = 0; pass < 2; pass++) { + const chain = this._flushFollowUp ?? this._flushInFlight + if (!chain) break + await chain.catch(() => {}) + } // Cancel any pending post-import background deduplication FIRST — it is a // writer (merge-deletes), and no delete pass may start mid- or post-close. From da9519903a3de52b2e0aeeb6e33a1257c6f5b749 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:42:04 -0700 Subject: [PATCH 103/150] =?UTF-8?q?test(shutdown):=20pin=20one=20owner=20p?= =?UTF-8?q?er=20brain=20=E2=80=94=20real=20processes,=20real=20signals?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four pins in real child processes under real SIGTERM, following the writer-lock-clean-close spawn pattern: (a) A host owner registered on SIGTERM closes two brains while the engine's hooks are live: exactly one close entered and one close body run per brain, the writer lock given up exactly ONCE per brain, the handler announcing that it stepped aside, no "Writer fence lost", no failed instance, both durability markers written, exit 0, and both reopens adopting rather than folding. The release count is the discriminating assertion — against the old handler it reads {a: 2, b: 2}, one release from the owner's close and one from the handler's own finally. (b) No host owner: the engine's handler closes every instance by the same path — one close each, markers written, clean exit, clean reopen. (c) Two concurrent close() callers share one promise (by identity) and one execution; a third call after they settle runs nothing. (d) Eight kicks during a running flush — five through the cadence door, three direct — arm exactly ONE follow-up: two flush bodies total, and the concurrency high-water mark stays at 1. The counts come out of the child through a file written synchronously on the way out: the engine calls process.exit(0) when it is the sole shutdown owner, and a console.log to a pipe can be dropped by that exit. --- .../integration/shutdown-single-owner.test.ts | 405 ++++++++++++++++++ 1 file changed, 405 insertions(+) create mode 100644 tests/integration/shutdown-single-owner.test.ts diff --git a/tests/integration/shutdown-single-owner.test.ts b/tests/integration/shutdown-single-owner.test.ts new file mode 100644 index 00000000..d3c02f99 --- /dev/null +++ b/tests/integration/shutdown-single-owner.test.ts @@ -0,0 +1,405 @@ +/** + * @module tests/integration/shutdown-single-owner + * @description ONE SHUTDOWN, ONE OWNER. + * + * MEASURED IN PRODUCTION. A host that owns its own shutdown — one SIGTERM + * listener calling `close()` on every pooled store — ran head-on into the + * engine's own signal handler, which iterated every live instance, flushed its + * components in parallel, and released its writer lock in a `finally`. Two + * teardowns of the same brain at the same moment. The log shape: + * + * "Shutdown signal received - flushing pending data..." (SIGTERM) + * ...148 seconds of silence... + * "Flushed successfully (1 instance)" + * ...the host's pool close of that same store returns 1s later + * + * 149s for the one store with engine work in flight, against 24s for its six + * idle siblings. The same race in a local reproduction printed + * `Failed to flush one Brainy instance on shutdown: Writer fence lost … the + * lock file is gone` — the handler observing a lock the close it was racing + * had already released. + * + * The contract pinned here: + * (a) A host owner and the engine's hooks both live: EXACTLY ONE close runs + * per brain, no fence is lost, both durability markers are written, the + * process exits 0, and the reopen adopts rather than folding. + * (b) No host owner: the engine's handler closes every instance by the same + * `close()` path — markers written, clean exit. + * (c) `close()` is idempotent and re-entrant: concurrent callers share ONE + * execution and all of them settle. + * (d) Flush is single-flight: N kicks during a running flush arm exactly one + * follow-up, and two flush bodies never overlap. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs' +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const REPO_ROOT = process.cwd() +const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx') +const BRAINY_SRC = join(REPO_ROOT, 'src', 'brainy.ts') + +function makeTempDir(prefix: string): string { + return mkdtempSync(join(tmpdir(), prefix)) +} + +/** The writer lock's clean-close record — written by `releaseWriterLock()`. */ +const closeRecordPath = (dir: string) => join(dir, 'locks', '_writer.close') +/** + * The generation store's clean-shutdown marker — the adopt-vs-fold gate. + * (`FileSystemStorage` gzips raw objects, so the file on disk carries `.gz`; + * both spellings are accepted so the pin survives a compression change.) + */ +const cleanShutdownWritten = (dir: string) => + existsSync(join(dir, '_system', 'clean-shutdown.json.gz')) || + existsSync(join(dir, '_system', 'clean-shutdown.json')) + +/** + * Write a child script and start it under tsx, in its OWN process group so a + * group-wide signal reaches the grandchild that actually holds the writer + * lock. (A file, not `tsx -e`: the eval form compiles to CommonJS, which has + * no top-level await.) + */ +function startChild(scriptDir: string, body: string): ReturnType { + const scriptPath = join(scriptDir, 'child-process.mts') + writeFileSync(scriptPath, body) + return spawn(TSX, [scriptPath], { + cwd: REPO_ROOT, + stdio: ['ignore', 'pipe', 'pipe'], + detached: true + }) +} + +/** Start a child and resolve once it prints READY, collecting all its output. */ +function startAndAwaitReady( + scriptDir: string, + body: string +): Promise<{ child: ReturnType; output: () => string }> { + const child = startChild(scriptDir, body) + let out = '' + child.stdout?.on('data', (d) => { out += String(d) }) + child.stderr?.on('data', (d) => { out += String(d) }) + return new Promise((resolvePromise, rejectPromise) => { + const timer = setTimeout( + () => rejectPromise(new Error(`child never became READY:\n${out}`)), + 120_000 + ) + child.stdout?.on('data', () => { + if (out.includes('READY')) { + clearTimeout(timer) + resolvePromise({ child, output: () => out }) + } + }) + child.on('exit', (code) => { + clearTimeout(timer) + if (!out.includes('READY')) rejectPromise(new Error(`child exited ${code} before READY:\n${out}`)) + }) + }) +} + +/** Capture console.warn/error/log lines emitted while `fn` runs. */ +async function captureConsole(fn: () => Promise): Promise<{ result: T; lines: string[] }> { + const lines: string[] = [] + const orig = { log: console.log, warn: console.warn, error: console.error } + const sink = (...args: unknown[]) => { lines.push(args.map((a) => String(a)).join(' ')) } + console.log = sink as typeof console.log + console.warn = sink as typeof console.warn + console.error = sink as typeof console.error + try { + return { result: await fn(), lines } + } finally { + console.log = orig.log + console.warn = orig.warn + console.error = orig.error + } +} + +/** + * Reopen a store and assert the open ADOPTED: no crash-recovery fold, no + * stale-lock verdict. This is the whole point of a close having run exactly + * once — a fold is measured in tens of seconds on a real store. + */ +async function expectCleanReopen(dir: string): Promise { + const { result, lines } = await captureConsole(async () => { + const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await next.init() + return next + }) + try { + expect(lines.filter((l) => /log-authority recovery|unclean shutdown detected/i.test(l))).toEqual([]) + expect(lines.filter((l) => /Overwriting stale writer lock|appears dead/i.test(l))).toEqual([]) + } finally { + await result.close() + } +} + +/** The child's counts of closes entered and close bodies run, per brain. */ +function readResult( + resultPath: string, + out: string +): { entries: Record; bodies: Record; releases: Record } { + if (!existsSync(resultPath)) throw new Error(`child wrote no result file:\n${out}`) + return JSON.parse(readFileSync(resultPath, 'utf-8')) +} + +/** + * The child-side instrumentation, shared by (a) and (b): count how many times + * `close()` is ENTERED per brain and how many times its body actually RUNS. + * The counting wrapper is an OWN property, so it shadows the prototype for + * every caller — including the engine's own signal handler, which calls + * `instance.close()`. + * + * `report()` writes SYNCHRONOUSLY to a file: it runs on the way out of the + * process (the engine's handler calls `process.exit(0)` when it is the sole + * shutdown owner), and a `console.log` to a pipe is asynchronous and can be + * dropped by that exit. + */ +function childCounters(resultPath: string): string { + return ` + const entries = {} + const bodies = {} + const releases = {} + function instrument(name, brain) { + entries[name] = 0 + bodies[name] = 0 + releases[name] = 0 + const enter = brain.close.bind(brain) + brain.close = () => { entries[name]++; return enter() } + const durable = brain.closeDurableSteps.bind(brain) + brain.closeDurableSteps = () => { bodies[name]++; return durable() } + // The writer lock is the ownership witness: the old handler released it + // in its own finally, on top of the owner's close doing the same. + const storage = brain.storage + const release = storage.releaseWriterLock.bind(storage) + storage.releaseWriterLock = () => { releases[name]++; return release() } + } + const report = () => { + __writeFileSync(${JSON.stringify(resultPath)}, JSON.stringify({ entries, bodies, releases })) + } +` +} + +describe('shutdown has exactly one owner', () => { + let dirA: string + let dirB: string + let scriptDir: string + let resultPath: string + + beforeEach(() => { + dirA = makeTempDir('brainy-shutdown-owner-a-') + dirB = makeTempDir('brainy-shutdown-owner-b-') + scriptDir = makeTempDir('brainy-shutdown-owner-script-') + resultPath = join(scriptDir, 'result.json') + }) + + afterEach(() => { + for (const d of [dirA, dirB, scriptDir]) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + it('(a) a host owner closes both brains and the engine handler steps aside', async () => { + const script = ` + import { writeFileSync as __writeFileSync } from 'node:fs' + import { Brainy } from ${JSON.stringify(BRAINY_SRC)} + const a = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dirA)} } }) + const b = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dirB)} } }) + await a.init() + await b.init() + await a.add({ data: 'row in brain a', type: 'concept' }) + await b.add({ data: 'row in brain b', type: 'concept' }) + ${childCounters(resultPath)} + instrument('a', a) + instrument('b', b) + // THE HOST'S OWN SHUTDOWN OWNER, registered after the engine's hooks — + // the ordinary shape: the pool was built before the signal wiring. + process.on('SIGTERM', async () => { + await Promise.all([a.close(), b.close()]) + // Stay alive a beat so the engine's deferred handler gets its turn and + // has to decide what to do about two already-closed brains. + await new Promise((r) => setTimeout(r, 1500)) + report() + process.exit(0) + }) + console.log('READY') + setInterval(() => {}, 1000) + ` + const { child, output } = await startAndAwaitReady(scriptDir, script) + + process.kill(-(child.pid as number), 'SIGTERM') + const code = await new Promise((r) => child.on('exit', (c) => r(c))) + // The tsx wrapper's exit event and the grandchild that actually held the + // locks are asynchronous with each other — let its last writes land. + await new Promise((r) => setTimeout(r, 750)) + const out = output() + + // The process shut down cleanly. + expect(code, `child output:\n${out}`).toBe(0) + + // EXACTLY ONE close per brain — entered once, body run once. A second + // entry would mean the engine's handler closed a brain its owner was + // already closing; a second body would mean close() is not single-flight. + const { entries, bodies, releases } = readResult(resultPath, out) + expect(entries).toEqual({ a: 1, b: 1 }) + expect(bodies).toEqual({ a: 1, b: 1 }) + // ...and the writer lock was given up exactly once per brain. This is the + // assertion that fails on the old handler, which released the lock in its + // own `finally` on top of the owner's close doing the same — two owners. + expect(releases).toEqual({ a: 1, b: 1 }) + + // The engine's handler ran (it announced the signal) and stepped aside for + // both brains rather than touching them. setImmediate lands in the check + // phase of the same loop turn, so a close that has begun cannot have + // finished — it is still in flight when the handler looks. + expect(out).toContain('Shutdown signal received') + expect(out).toMatch(/2 Brainy instances are already closing/) + + // Nothing was taken out from under the owner, and nothing failed. + expect(out).not.toMatch(/Writer fence lost/i) + expect(out).not.toMatch(/Failed to (flush|close) one Brainy instance/i) + + // Both durability markers, both brains: the writer lock's clean-close + // record and the generation store's clean-shutdown marker. + for (const dir of [dirA, dirB]) { + expect(existsSync(closeRecordPath(dir)), `clean-close record missing in ${dir}`).toBe(true) + expect(cleanShutdownWritten(dir), `clean-shutdown marker missing in ${dir}`).toBe(true) + } + + // And the next open adopts instead of folding. + await expectCleanReopen(dirA) + await expectCleanReopen(dirB) + }, 240_000) + + it('(b) with no host owner the engine closes every instance the same way', async () => { + const script = ` + import { writeFileSync as __writeFileSync } from 'node:fs' + import { Brainy } from ${JSON.stringify(BRAINY_SRC)} + const a = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dirA)} } }) + const b = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dirB)} } }) + await a.init() + await b.init() + await a.add({ data: 'row in brain a', type: 'concept' }) + await b.add({ data: 'row in brain b', type: 'concept' }) + ${childCounters(resultPath)} + instrument('a', a) + instrument('b', b) + process.on('exit', report) + console.log('READY') + setInterval(() => {}, 1000) + ` + const { child, output } = await startAndAwaitReady(scriptDir, script) + + process.kill(-(child.pid as number), 'SIGTERM') + const code = await new Promise((r) => child.on('exit', (c) => r(c))) + // The tsx wrapper's exit event and the grandchild that actually held the + // locks are asynchronous with each other — let its last writes land. + await new Promise((r) => setTimeout(r, 750)) + const out = output() + + expect(code, `child output:\n${out}`).toBe(0) + + // The engine owned this shutdown: one close per brain, through close(). + const { entries, bodies, releases } = readResult(resultPath, out) + expect(entries).toEqual({ a: 1, b: 1 }) + expect(bodies).toEqual({ a: 1, b: 1 }) + expect(releases).toEqual({ a: 1, b: 1 }) + expect(out).toContain('Shutdown signal received') + expect(out).toMatch(/Flushed successfully \(2 instances\)/) + expect(out).not.toMatch(/Writer fence lost/i) + expect(out).not.toMatch(/Failed to (flush|close) one Brainy instance/i) + + for (const dir of [dirA, dirB]) { + expect(existsSync(closeRecordPath(dir)), `clean-close record missing in ${dir}`).toBe(true) + expect(cleanShutdownWritten(dir), `clean-shutdown marker missing in ${dir}`).toBe(true) + } + + await expectCleanReopen(dirA) + await expectCleanReopen(dirB) + }, 240_000) + + it('(c) two concurrent close() callers share ONE execution, and both settle', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dirA } }) + await brain.init() + await brain.add({ data: 'one row', type: NounType.Concept }) + + const inner = brain as unknown as { closeDurableSteps: () => Promise } + const durable = inner.closeDurableSteps.bind(inner) + let bodies = 0 + inner.closeDurableSteps = () => { bodies++; return durable() } + + expect(brain.isClosing).toBe(false) + expect(brain.isClosed).toBe(false) + + const first = brain.close() + // The state is observable IMMEDIATELY — a signal handler that yields a + // tick and comes back must not read a stale "not yet". + expect(brain.isClosing).toBe(true) + const second = brain.close() + expect(first === second, 'concurrent callers must share the one promise').toBe(true) + + await Promise.all([first, second]) + expect(bodies).toBe(1) + expect(brain.isClosed).toBe(true) + + // A caller arriving after the close finished gets the same settled answer, + // and nothing runs again. + await brain.close() + expect(bodies).toBe(1) + + expect(existsSync(closeRecordPath(dirA))).toBe(true) + expect(cleanShutdownWritten(dirA)).toBe(true) + }, 120_000) + + it('(d) N kicks during a running flush arm exactly one follow-up, never a second flush', async () => { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dirA } }) + await brain.init() + + const inner = brain as unknown as { + _flushBodyRuns: number + _flushConcurrencyPeak: number + _flushInFlight: Promise | null + _flushFollowUp: Promise | null + _persistBackgroundFlight: Promise | null + metadataIndex: { flush: () => Promise } + kickBackgroundFlush: (reason: 'threshold' | 'idle') => void + } + + // Widen the flush body's window so the kicks land INSIDE it — the + // production shape, where two flushes overlapped 3s apart. + const metaFlush = inner.metadataIndex.flush.bind(inner.metadataIndex) + inner.metadataIndex.flush = async () => { + await new Promise((r) => setTimeout(r, 400)) + return metaFlush() + } + + await brain.add({ data: 'a write to flush', type: NounType.Concept }) + const runsBefore = inner._flushBodyRuns + + const leader = brain.flush() + await new Promise((r) => setTimeout(r, 50)) // the leader is inside its body + expect(inner._flushInFlight, 'a flush is running').not.toBeNull() + + // The cadence kicks — the door named in the defect — plus direct callers + // (an application flush, the cross-process flush-request watcher). + for (let i = 0; i < 5; i++) inner.kickBackgroundFlush('threshold') + const direct = [brain.flush(), brain.flush(), brain.flush()] + + // EXACTLY ONE follow-up is armed, however many callers arrived. + expect(inner._flushFollowUp, 'the eight kicks armed one follow-up').not.toBeNull() + + await Promise.all([leader, ...direct, inner._persistBackgroundFlight ?? Promise.resolve()]) + + // One leader + one follow-up. Not nine, and never two at once. + expect(inner._flushBodyRuns - runsBefore).toBe(2) + expect(inner._flushConcurrencyPeak).toBe(1) + expect(inner._flushInFlight).toBeNull() + expect(inner._flushFollowUp).toBeNull() + + inner.metadataIndex.flush = metaFlush + await brain.close() + }, 120_000) +}) From a79db434acbc1b3476ca97b979e291375af9bf86 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:53:54 -0700 Subject: [PATCH 104/150] =?UTF-8?q?fix(generation-store):=20commitTransact?= =?UTF-8?q?ion=20refuses=20while=20single-ops=20are=20pending=20=E2=80=94?= =?UTF-8?q?=20the=20order=20invariant=20is=20enforced,=20not=20assumed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reservedGensAsc() documented but never enforced that pending single-op generations must sort above every committed one. A direct commitTransaction() call bypassing Brainy.transact()'s flush-first step could commit a fresh generation into committedRanges above lower, still-pending ones, unsorting the committed-then-pending walk resolveManyAt relies on and returning a wrong before-image for a point-in-time read — silently. commitTransaction() now refuses via a new PendingSingleOpsUnflushedError when the pending tier is non-empty, before any staging I/O. Behavior-neutral: both sanctioned callers (Brainy.transact(), Brainy.compactHistory()) already flush first. --- src/db/errors.ts | 60 +++++ src/db/generationStore.ts | 56 +++- src/index.ts | 3 +- .../db/generationStore-commit-guard.test.ts | 254 ++++++++++++++++++ 4 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 tests/unit/db/generationStore-commit-guard.test.ts diff --git a/src/db/errors.ts b/src/db/errors.ts index 3b4c1af6..da62eb0b 100644 --- a/src/db/errors.ts +++ b/src/db/errors.ts @@ -351,3 +351,63 @@ export class PendingFlushDurabilityError extends Error { this.failedAttempts = failedAttempts } } + +/** + * @description Thrown by {@link GenerationStore.commitTransaction} when the + * PENDING single-op tier is non-empty — i.e. one or more `commitSingleOp()` + * generations are buffered in memory, not yet flushed to + * `committedRanges` via `flushPendingSingleOps()`. + * + * The invariant `reservedGensAsc()` (and everything built on it — + * `resolveManyAt`, `resolveAt`, `changedBetween`, the hot-tail window) relies + * on is documented, not enforced by types: pending generations must always be + * numerically greater than every committed one, because the ONLY sanctioned + * callers of `commitTransaction()` — `Brainy.transact()` and + * `Brainy.compactHistory()` — flush the pending tier FIRST. A caller that + * invokes `commitTransaction()` directly while single-ops are still pending + * breaks that invariant: the new commit lands in `committedRanges` ABOVE + * generations still sitting in `pendingGens`, so the committed-then-pending + * concatenation `reservedGensAsc()` yields is no longer ascending. The + * concrete failure this produces is silent, not a crash: `resolveManyAt` + * walks committed ranges before pending ones, so it can report a NEWER + * generation as the "first after" a pin than an older, still-pending one that + * actually touched the id first — a wrong before-image at a point-in-time + * read, without a compensating error to warn a caller anything went wrong. + * + * This error refuses the commit outright, before any staging I/O: nothing is + * written, the generation counter reservation is untouched, and + * `committedRanges`/`pendingGens` are exactly as they were. Call + * `flushPendingSingleOps()` first (or go through `Brainy.transact()`, which + * already does). + * + * @example + * try { + * await generationStore.commitTransaction({ touched, execute }) + * } catch (err) { + * if (err instanceof PendingSingleOpsUnflushedError) { + * await generationStore.flushPendingSingleOps() + * await generationStore.commitTransaction({ touched, execute }) // now safe + * } + * } + */ +export class PendingSingleOpsUnflushedError extends Error { + /** How many un-flushed single-op generations were buffered at refusal time. */ + public readonly pendingCount: number + + /** + * @param pendingCount - `pendingGens.length` at the moment of refusal (always ≥ 1). + */ + constructor(pendingCount: number) { + super( + `commitTransaction() refused: ${pendingCount} pending single-op generation(s) ` + + `are still buffered and un-flushed. Flush the pending single-op tier before ` + + `committing a transaction — Brainy.transact() does this automatically; a ` + + `direct commitTransaction() call with pending generations would leave the ` + + `generation order unsorted (committed generations landing above lower, ` + + `still-pending ones) and make point-in-time reads (resolveManyAt/resolveAt) ` + + `return the wrong before-image. Call flushPendingSingleOps() first, then retry.` + ) + this.name = 'PendingSingleOpsUnflushedError' + this.pendingCount = pendingCount + } +} diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index da21dc61..128f905b 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -32,7 +32,13 @@ */ import { prodLog } from '../utils/logger.js' -import { GenerationCompactedError, GenerationConflictError, PendingFlushDurabilityError, StoreInconsistentError } from './errors.js' +import { + GenerationCompactedError, + GenerationConflictError, + PendingFlushDurabilityError, + PendingSingleOpsUnflushedError, + StoreInconsistentError +} from './errors.js' import type { UnreconciledRecord } from './errors.js' import { TransactionRollbackError } from '../transaction/errors.js' import type { @@ -1351,6 +1357,9 @@ export class GenerationStore { * @param args.execute - Runs the planned operation batch atomically. * @returns The committed generation and its commit timestamp. * @throws GenerationConflictError when the CAS expectation fails. + * @throws PendingSingleOpsUnflushedError when the pending single-op tier is + * non-empty — call `flushPendingSingleOps()` first (both `Brainy.transact()` + * and `Brainy.compactHistory()` already do). */ /** * The generation fact log, or `null` when the storage layer cannot host one. @@ -1425,6 +1434,13 @@ export class GenerationStore { execute: () => Promise }): Promise<{ generation: number; timestamp: number }> { return this.withMutex(async () => { + // The generation-order guard (see assertPendingSingleOpsFlushed): a + // direct commitTransaction() call while single-ops are still pending + // would commit above them, unsorting reservedGensAsc() and corrupting + // point-in-time reads. Both sanctioned callers (Brainy.transact(), + // Brainy.compactHistory()) already flush first, so this is + // behavior-neutral on every real path. + this.assertPendingSingleOpsFlushed() // A latched history-durability failure compromises the whole generation // chain — refuse a transact too (advancing the manifest past stuck, // un-durable single-op generations would be inconsistent). Same loud @@ -2294,6 +2310,37 @@ export class GenerationStore { } } + /** + * @description Throw if the pending single-op tier is non-empty. Called at + * the top of {@link commitTransaction} (the ONLY method that appends a + * fresh commit directly into {@link committedRanges} outside recovery) so + * the ordering invariant {@link reservedGensAsc}'s own doc comment states — + * "pending generations are always greater than every committed one" — is + * ENFORCED there rather than merely assumed. + * + * That invariant holds today only because both sanctioned callers flush the + * pending tier before committing: `Brainy.transact()` (src/brainy.ts, + * `await this.generationStore.flushPendingSingleOps()` immediately before + * its `commitTransaction()` call) and `Brainy.compactHistory()` + * (src/brainy.ts, the same flush immediately before its `compact()` call — + * `compact()` itself only ever RECLAIMS an existing committed prefix, so it + * cannot land a commit out of order and needs no guard of its own). A + * caller that reaches `commitTransaction()` by any other path — bypassing + * that flush — would commit a new generation into `committedRanges` ABOVE + * generations still sitting in `pendingGens`, breaking `reservedGensAsc`'s + * "committed-then-pending is already sorted" assumption and making + * `resolveManyAt`'s single ascending pass (and `resolveAt`'s consumers) + * return the WRONG before-image for a point-in-time read — silently, no + * compensating error. Refusing here, before any staging I/O, keeps the + * store untouched (nothing committed, nothing staged, the generation + * counter reservation unaffected) on every path that already flushes. + */ + private assertPendingSingleOpsFlushed(): void { + if (this.pendingGens.length > 0) { + throw new PendingSingleOpsUnflushedError(this.pendingGens.length) + } + } + /** Schedule a coalesced pending-tier flush (size trigger fires immediately on * the next microtask; otherwise a {@link PENDING_FLUSH_DELAY_MS} timer). Both * defer outside the current mutex section so the flush can re-acquire it. A @@ -2377,6 +2424,13 @@ export class GenerationStore { * committed-then-pending concatenation is already sorted — identical to the old * `[...committedGens, ...pendingGens]`. This is the union historical reads * resolve over so un-flushed single-ops are visible to pins/`asOf`. + * + * The "flush first" half of that invariant is ENFORCED, not just documented: + * {@link commitTransaction} — the only method that lands a fresh commit into + * {@link committedRanges} outside crash recovery — refuses via + * {@link assertPendingSingleOpsFlushed} whenever {@link pendingGens} is + * non-empty, so a committed generation can never land above a still-pending + * one and break this ordering. */ private *reservedGensAsc(): IterableIterator { yield* this.committedGensAsc() diff --git a/src/index.ts b/src/index.ts index edc21809..e946f15c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -231,7 +231,8 @@ export { GenerationCompactedError, StoreInconsistentError, PendingFlushDurabilityError, - CanonicalEnumerationUnavailableError + CanonicalEnumerationUnavailableError, + PendingSingleOpsUnflushedError } from './db/errors.js' export type { UnreconciledRecord } from './db/errors.js' export type { diff --git a/tests/unit/db/generationStore-commit-guard.test.ts b/tests/unit/db/generationStore-commit-guard.test.ts new file mode 100644 index 00000000..d449f8ef --- /dev/null +++ b/tests/unit/db/generationStore-commit-guard.test.ts @@ -0,0 +1,254 @@ +/** + * @module tests/unit/db/generationStore-commit-guard + * @description Pins the commit-order guard on + * `GenerationStore.commitTransaction()` (`src/db/generationStore.ts`). + * + * `reservedGensAsc()`'s own doc comment states an invariant it never + * enforced: pending single-op generations are always greater than every + * committed one, because the store's only two sanctioned callers — + * `Brainy.transact()` and `Brainy.compactHistory()` — flush the pending tier + * before committing. Nothing stopped a caller from invoking + * `commitTransaction()` directly while single-ops were still buffered: the + * fresh commit would land in `committedRanges` ABOVE those lower, + * still-pending generations, so the committed-then-pending concatenation + * `reservedGensAsc()` yields is no longer ascending — and `resolveManyAt` + * (which walks committed ranges before pending ones) would silently report a + * WRONG before-image for a point-in-time read. `commitTransaction()` now + * refuses loudly (`PendingSingleOpsUnflushedError`) instead of assuming. + * + * Four pins: + * 1. A direct `commitTransaction()` call while single-ops are pending throws + * and commits NOTHING. + * 2. The same commit succeeds once the pending tier is flushed first. + * 3. `Brainy.transact()` — which already flushes first — is unaffected + * (mirrors `tests/unit/db/generation-chain.test.ts`'s `seedX()`/`bumpX()` + * transact pin: add, then transact-update, generation advances by one + * each time, the update lands). + * 4. `reservedGensAsc()` stays ascending across a real add+transact+delete + * workload — proven by point-in-time reads (`asOf`) staying correct + * throughout, which is exactly what an ordering break would corrupt. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js' +import { + GenerationStore, + GENERATIONS_PREFIX, + MANIFEST_PATH +} from '../../../src/db/generationStore.js' +import { PendingSingleOpsUnflushedError } from '../../../src/db/errors.js' +import { Brainy } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' +import { createTestConfig, generateTestVector } from '../../helpers/test-factory.js' + +/** Precomputed embedding so Brainy-level adds skip the (slow) embedding model — + * these tests exercise the generation layer, not semantics. */ +const VEC = generateTestVector() + +// Entity ids must be UUID-shaped (the sharded storage layout derives the +// shard from the UUID hex) — same fixture convention as generationStore.test.ts. +const ID_A = '00000000-0000-4000-8000-0000000000aa' +const ID_B = '00000000-0000-4000-8000-0000000000bb' + +/** Stored-metadata fixture in the canonical shape the live write paths use + * (matches generationStore.test.ts's fixture exactly). */ +function metadataFixture(version: number): Record { + return { + noun: NounType.Document, + subtype: 'note', + data: `payload-v${version}`, + version, + createdAt: 1000, + updatedAt: 1000 + version, + _rev: version + } +} + +describe('db/GenerationStore — commitTransaction pending-tier guard (store level)', () => { + let storage: MemoryStorage + let store: GenerationStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + store = new GenerationStore(storage) + await store.open() + }) + + /** Buffer one single-op generation via commitSingleOp WITHOUT flushing — + * the pending tier that must be drained before commitTransaction(). */ + async function pendingSingleOp(id: string, version: number): Promise { + const { generation } = await store.commitSingleOp({ + touched: { nouns: [id] }, + execute: async () => { + await storage.saveNounMetadata(id, metadataFixture(version)) + } + }) + return generation + } + + /** A direct transact commit — exactly what a caller bypassing + * Brainy.transact()'s flush-first step would issue. */ + function directCommit(id: string, version: number): Promise<{ generation: number; timestamp: number }> { + return store.commitTransaction({ + touched: { nouns: [id], verbs: [] }, + execute: async () => { + await storage.saveNounMetadata(id, metadataFixture(version)) + } + }) + } + + it('PIN 1: refuses a direct commitTransaction() while single-ops are pending, and commits NOTHING', async () => { + const g1 = await pendingSingleOp(ID_A, 1) + expect(g1).toBe(1) + expect(store.committedGeneration()).toBe(0) // nothing flushed to disk yet + + let caught: unknown + try { + await directCommit(ID_B, 1) + expect.unreachable('should have thrown PendingSingleOpsUnflushedError') + } catch (err) { + caught = err + } + expect(caught).toBeInstanceOf(PendingSingleOpsUnflushedError) + expect((caught as PendingSingleOpsUnflushedError).pendingCount).toBe(1) + + // Nothing committed: the head + committed ranges are unchanged, and the + // counter never advanced for the refused attempt (the guard fires before + // a generation is even reserved). + expect(store.committedGeneration()).toBe(0) + expect(store.generation()).toBe(1) // still just the pending single-op's gen + expect(await storage.readRawObject(MANIFEST_PATH)).toBeNull() + // The guard fires BEFORE a generation is reserved (`gen = ++this.counter` + // never runs), so the refused attempt's would-be directory (generation 2, + // the next number after the pending single-op's 1) was never created. + expect(await storage.listRawObjects(`${GENERATIONS_PREFIX}/2`)).toEqual([]) + + // The refused write never touched canonical storage. + expect((await storage.readNounRaw(ID_B)).metadata).toBeNull() + + // The pending tier itself is untouched by the refused attempt — flushing + // now still commits the ORIGINAL single-op cleanly. + await store.flushPendingSingleOps() + expect(store.committedGeneration()).toBe(1) + const atG0 = await store.resolveAt('noun', ID_A, 0) + expect(atG0).toEqual({ source: 'absent' }) // the create sentinel before g1's write + }) + + it('PIN 2: the same commit succeeds once the pending tier is flushed first', async () => { + await pendingSingleOp(ID_A, 1) + await expect(directCommit(ID_B, 1)).rejects.toBeInstanceOf(PendingSingleOpsUnflushedError) + + await store.flushPendingSingleOps() + expect(store.committedGeneration()).toBe(1) + + const { generation } = await directCommit(ID_B, 1) + expect(generation).toBe(2) + expect(store.committedGeneration()).toBe(2) + expect((await storage.readNounRaw(ID_B)).metadata).toMatchObject({ version: 1 }) + }) +}) + +describe('Brainy public API — commitTransaction pending-tier guard is behavior-neutral', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + afterEach(async () => { + await brain.close() + }) + + it('PIN 3: Brainy.transact() still commits normally over pending single-ops (mirrors generation-chain.test.ts\'s seedX()/bumpX() transact pin)', async () => { + const store = (brain as any).generationStore as GenerationStore + // Relative, not absolute: under the adopt-at-open default the open-time + // baseline backfill takes a generation of its own (see + // bounded-chains.test.ts's identical note), so the first user add is not + // necessarily generation 1. + const baseGen = brain.generation() + const baseCommitted = store.committedGeneration() + + const id = await brain.add({ + data: 'x', + type: NounType.Document, + subtype: 'note', + metadata: { v: 1 }, + vector: VEC + }) + // The add is a pending single-op generation — NOT yet flushed. + expect(brain.generation()).toBe(baseGen + 1) + expect(store.committedGeneration()).toBe(baseCommitted) + + // Brainy.transact() flushes the pending tier FIRST (src/brainy.ts: + // `await this.generationStore.flushPendingSingleOps()`, immediately + // before its `generationStore.commitTransaction()` call), so the guard + // never fires on this path — same shape as generation-chain.test.ts's + // seedX() (add) → bumpX() (transact update) → generation advances by one. + const db = await brain.transact([{ op: 'update', id, metadata: { v: 2 } }]) + await db.release() + + expect(brain.generation()).toBe(baseGen + 2) + expect(store.committedGeneration()).toBe(baseGen + 2) // the flushed add + the transact update + const entity = (await brain.get(id)) as any + expect(entity.metadata.v).toBe(2) + }) + + it('PIN 4: reservedGensAsc() stays ascending across a real add+transact+delete workload — point-in-time reads stay correct', async () => { + const store = (brain as any).generationStore as GenerationStore + const baseGen = brain.generation() + const baseCommitted = store.committedGeneration() + + const idX = await brain.add({ + data: 'x', + type: NounType.Document, + subtype: 'note', + metadata: { v: 1 }, + vector: VEC + }) + expect(brain.generation()).toBe(baseGen + 1) // pending (un-flushed) + + const idY = await brain.add({ + data: 'y', + type: NounType.Document, + subtype: 'note', + metadata: { v: 1 }, + vector: VEC + }) + // Pin right after BOTH adds — before the transact update — so X reads v1 + // and Y still exists at this pin, unlike the live head after the rest of + // the workload runs. + const pinAfterBothAdds = brain.generation() + expect(pinAfterBothAdds).toBe(baseGen + 2) // ALSO pending — two un-flushed single-ops + expect(store.committedGeneration()).toBe(baseCommitted) + + // A transact() flushes baseGen+1 and baseGen+2 first, then commits its + // own update as baseGen+3. If committed-vs-pending ordering ever broke, + // this is exactly the step that would land a commit ABOVE still-pending + // generations. + const db = await brain.transact([{ op: 'update', id: idX, metadata: { v: 3 } }]) + await db.release() + expect(brain.generation()).toBe(baseGen + 3) + expect(store.committedGeneration()).toBe(baseGen + 3) + + // A single-op delete, pending again (un-flushed). + await brain.remove(idY) + expect(brain.generation()).toBe(baseGen + 4) + + // A point-in-time read pinned right after the two adds (before the + // transact update) must see X's PRE-update value and Y still present. + // This is precisely what resolveManyAt/resolveAt get WRONG if committed + // and pending generations were ever interleaved out of ascending order. + const past = await brain.asOf(pinAfterBothAdds) + const xAtPin = (await past.get(idX)) as any + expect(xAtPin?.metadata?.v).toBe(1) + const yAtPin = (await past.get(idY)) as any + expect(yAtPin?.metadata?.v).toBe(1) // not yet removed, as of this pin + await past.release() + + // Live state reflects every later write, in the right order. + const xNow = (await brain.get(idX)) as any + expect(xNow.metadata.v).toBe(3) + expect(await brain.get(idY)).toBeNull() + }) +}) From 367ca721a5d7dd9b711e9ec7c83d155166986b71 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 10:56:32 -0700 Subject: [PATCH 105/150] =?UTF-8?q?fix(close):=20a=20read-only=20brain=20w?= =?UTF-8?q?rites=20no=20clean-shutdown=20evidence=20=E2=80=94=20the=20mark?= =?UTF-8?q?er=20is=20the=20writer's=20word=20about=20itself?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/brainy.ts | 17 +- src/db/generationStore.ts | 17 +- .../readonly-close-no-marker.test.ts | 250 ++++++++++++++++++ 3 files changed, 280 insertions(+), 4 deletions(-) create mode 100644 tests/integration/readonly-close-no-marker.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 39b604ad..02f2ca3d 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -20486,9 +20486,22 @@ export class Brainy implements BrainyInterface { await this._aggregationIndex.flush() } })(), - // 8.0 MVCC: detach the generation-bump hook and persist the counter + // 8.0 MVCC: detach the generation-bump hook and persist the counter. + // READ-ONLY GUARD: a reader's open() never sets the bump hook, never + // buffers pending single-ops, and — since generationStore.open() also + // leaves the clean-shutdown marker untouched for a reader — never + // consumes it either, so there is nothing of a writer's to persist or + // release here. Calling close() anyway would still WRITE: it + // unconditionally re-stamps `_system/clean-shutdown.json` (and can + // advance the fold checkpoint / counter files) at the generation this + // session merely observed — a reader vouching for a commit it never + // made. The marker is the writer's own evidence about the writer's own + // process; a read-only brain must leave `_system/` exactly as it found + // it. (Mirrors the same guard already applied to every other Phase-1 + // step below, and to the signal-path shutdown in + // registerShutdownHooks().) (async () => { - if (this.generationStore) { + if (this.generationStore && !this.isReadOnly) { await this.generationStore.close() } })() diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 128f905b..89f83a8f 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -805,7 +805,16 @@ export class GenerationStore { if (uncleanOpen) await this.advanceFoldCheckpointUnlocked() // The marker is consumed: any session that can write invalidates it // at first commit (see the commit paths); a clean close re-writes it. - await this.clearCleanShutdownMarker() + // A READER NEVER CONSUMES IT. The marker is the writer's own evidence + // about the writer's own process — clearing it here exists so that + // if THIS session goes on to write and then dies before its next + // clean close, the marker's absence correctly reads as unclean. A + // reader can never write, so it can never leave the store in a state + // its own crash would mis-describe; clearing the marker for it would + // only cost the store's actual writer a needless whole-log fold on + // its next open, for a generation the reader merely observed. Leave + // `_system/` exactly as found. + if (!options?.readOnly) await this.clearCleanShutdownMarker() } await this.factLog.open(this.committed) } else { @@ -895,7 +904,11 @@ export class GenerationStore { } } - /** Consume the clean-shutdown marker (every open; a clean close re-writes it). */ + /** + * Consume the clean-shutdown marker (every WRITER open; a clean close + * re-writes it). Callers must gate this on `!options.readOnly` — a reader + * never consumes the marker, see the call site in {@link open}. + */ private async clearCleanShutdownMarker(): Promise { try { await this.storage.deleteRawObject(CLEAN_SHUTDOWN_PATH) diff --git a/tests/integration/readonly-close-no-marker.test.ts b/tests/integration/readonly-close-no-marker.test.ts new file mode 100644 index 00000000..7bcf99df --- /dev/null +++ b/tests/integration/readonly-close-no-marker.test.ts @@ -0,0 +1,250 @@ +/** + * @module tests/integration/readonly-close-no-marker + * @description A READ-ONLY BRAIN WRITES NO CLEAN-SHUTDOWN EVIDENCE. + * + * `_system/clean-shutdown.json` is the WRITER's own word about the writer's + * own process: "everything above this line, from THIS session, is durable." + * Two call sites treated a reader exactly like a writer: + * + * 1. `Brainy#closeDurableSteps()` called `generationStore.close()` + * unconditionally — a reader's close re-stamped the marker at the + * generation the reader merely OBSERVED, never committed. + * 2. `GenerationStore#open()` consumed (deleted) the marker on every open, + * reader or writer alike, so a reader that never got to a matching + * close left the store looking crashed to the next writer. + * + * Both are fixed by making a read-only brain leave `_system/` exactly as it + * found it — at open AND at close. Pinned here: + * + * 1. `_system/` is byte-for-byte identical (file set + contents) before and + * after a reader opens a cleanly-closed store, reads it, and closes. + * 2. After the reader's close, the next WRITER open adopts the marker as + * clean — no recovery fold narrates. + * 3. A reader creates no file under `_system/` merely by opening (before it + * ever closes). + * 4. A reader that opens and is then abandoned (crash-style, no close) does + * not force the next writer to pay a recovery fold — the concrete harm + * the fix closes. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' +import { abandonAsCrashed } from '../helpers/durabilityKillMatrix.js' + +function makeTempDir(): string { + return mkdtempSync(join(tmpdir(), 'brainy-readonly-close-')) +} + +/** Recursively hash every regular file under `dir`, keyed by its path relative to `dir`. */ +function snapshotDir(dir: string): Map { + const out = new Map() + const walk = (rel: string): void => { + const abs = rel ? join(dir, rel) : dir + let entries: string[] + try { + entries = readdirSync(abs) + } catch { + return + } + for (const name of entries) { + const childRel = rel ? join(rel, name) : name + const childAbs = join(dir, childRel) + const st = statSync(childAbs) + if (st.isDirectory()) { + walk(childRel) + } else if (st.isFile()) { + const hash = createHash('sha256').update(readFileSync(childAbs)).digest('hex') + out.set(childRel, hash) + } + } + } + walk('') + return out +} + +/** Capture console.warn lines (the narration channel — see `prodLog.narrate`) while `fn` runs. */ +async function captureWarn(fn: () => Promise): Promise<{ result: T; lines: string[] }> { + const lines: string[] = [] + const orig = console.warn + console.warn = ((...args: unknown[]) => { + lines.push(args.map((a) => String(a)).join(' ')) + }) as typeof console.warn + try { + return { result: await fn(), lines } + } finally { + console.warn = orig + } +} + +describe('a read-only brain writes no clean-shutdown evidence', () => { + let dir: string + let brain: Brainy | null = null + + beforeEach(() => { + dir = makeTempDir() + }) + + afterEach(async () => { + if (brain) { + try { + await brain.close() + } catch { + /* already closed */ + } + brain = null + } + try { + rmSync(dir, { recursive: true, force: true }) + } catch { + /* ignore */ + } + }) + + const systemDir = () => join(dir, '_system') + /** + * The marker file's actual on-disk name — `clean-shutdown.json` or, under + * FileSystemStorage's default gzip compression, `clean-shutdown.json.gz`. + * Returns null when absent. + */ + const findMarkerPath = (): string | null => { + let entries: string[] + try { + entries = readdirSync(systemDir()) + } catch { + return null + } + const name = entries.find((n) => n.startsWith('clean-shutdown.json')) + return name ? join(systemDir(), name) : null + } + + it('leaves `_system/`\'s file set and the clean-shutdown marker\'s bytes identical across a reader open → read → close', async () => { + // A writer opens, writes, and closes cleanly — the marker lands at + // whatever generation the writer actually committed. + const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer.init() + await writer.add({ data: 'seed entity', type: NounType.Concept }) + await writer.add({ data: 'second entity', type: NounType.Concept }) + await writer.flush() + await writer.close() + + const markerBeforePath = findMarkerPath() + expect(markerBeforePath, 'the writer left a clean-shutdown marker').not.toBeNull() + const before = snapshotDir(systemDir()) + expect(before.size).toBeGreaterThan(0) + const markerBeforeHash = before.get( + (markerBeforePath as string).slice(systemDir().length + 1) + ) + expect(markerBeforeHash).toBeTruthy() + + // A reader opens the same store, reads, and closes. + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + expect(brain.isReadOnly).toBe(true) + await brain.stats() + await brain.close() + brain = null + + // The FILE SET under `_system/` is unchanged — a reader creates and + // removes nothing. (Other files under `_system/` — e.g. the metadata + // field registry, which stamps its own `lastUpdated` on every persist — + // are a pre-existing, separate concern outside this fix's scope: this + // pin is specifically about the generation store's clean-shutdown + // evidence, not about every subsystem's close() being a true no-op for + // a reader.) + const after = snapshotDir(systemDir()) + expect([...after.keys()].sort()).toEqual([...before.keys()].sort()) + + // The MARKER's bytes are byte-for-byte identical — the reader neither + // consumed it at open nor re-stamped it at close. + const markerAfterPath = findMarkerPath() + expect(markerAfterPath, 'the marker must still exist, under the same name').toBe(markerBeforePath) + const markerAfterHash = after.get((markerAfterPath as string).slice(systemDir().length + 1)) + expect(markerAfterHash).toBe(markerBeforeHash) + }, 120_000) + + it('creates no file under `_system/` merely by opening read-only', async () => { + const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer.init() + await writer.add({ data: 'seed entity', type: NounType.Concept }) + await writer.flush() + await writer.close() + + const baselineNames = [...snapshotDir(systemDir()).keys()].sort() + expect(baselineNames.length).toBeGreaterThan(0) + + // Open the reader and inspect `_system/` BEFORE it ever closes — open() + // alone must create nothing. + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + const whileOpenNames = [...snapshotDir(systemDir()).keys()].sort() + expect(whileOpenNames).toEqual(baselineNames) + + await brain.close() + brain = null + }, 120_000) + + it('a writer reopening after the reader closes adopts the marker — no recovery fold', async () => { + const writer1 = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer1.init() + await writer1.add({ data: 'seed entity', type: NounType.Concept }) + await writer1.flush() + await writer1.close() + + // A reader opens and closes in between — must not disturb the marker. + const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await reader.stats() + await reader.close() + + // The next writer open must be a clean, no-fold open: no + // "log-authority recovery" / "WHOLE-LOG fold" narration line. + const { result: writer2, lines } = await captureWarn(async () => { + const w = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await w.init() + return w + }) + brain = writer2 + + const foldLines = lines.filter((l) => /log-authority recovery|WHOLE-LOG fold|recovery fold/i.test(l)) + expect(foldLines, `unexpected recovery narration:\n${foldLines.join('\n')}`).toEqual([]) + + // And the store is exactly what the first writer left — the seed row is + // still there, nothing was rolled back or re-derived. + const found = await writer2.find({ where: {} } as any) + expect(found.length).toBeGreaterThanOrEqual(1) + }, 120_000) + + it('a reader that opens and is then abandoned (never closes) does not force the next writer to fold', async () => { + // This is the concrete harm the fix closes: pre-fix, a reader's open() + // unconditionally DELETED the marker (consuming it as if it were the + // writer). A reader that opened and then died — no close, exactly like + // a killed process — left the marker gone, so the actual writer's next + // open read the store as crashed and paid a full recovery fold for a + // "crash" that was really just a reader that came and went. + const writer1 = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer1.init() + await writer1.add({ data: 'seed entity', type: NounType.Concept }) + await writer1.flush() + await writer1.close() + + const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await reader.stats() + // NEVER calls reader.close() — abandon it exactly like a killed process. + await abandonAsCrashed(reader) + + const { result: writer2, lines } = await captureWarn(async () => { + const w = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await w.init() + return w + }) + brain = writer2 + + const foldLines = lines.filter((l) => /log-authority recovery|WHOLE-LOG fold|recovery fold/i.test(l)) + expect( + foldLines, + `an abandoned READER forced a recovery fold on the next writer open:\n${foldLines.join('\n')}` + ).toEqual([]) + }, 120_000) +}) From 4142f36872f20d9dfafaec47474e5894528ec553 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 11:00:32 -0700 Subject: [PATCH 106/150] chore(contract): emit the 10.4.11 manifest 302 doors (17 added, executeGraphSearch removed), 7 error classes, 25 operators (4 refused by the index path). --check verified green against this candidate tip. --- docs/api-contract.json | 100 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 6 deletions(-) diff --git a/docs/api-contract.json b/docs/api-contract.json index aafd838a..12cb37c8 100644 --- a/docs/api-contract.json +++ b/docs/api-contract.json @@ -161,6 +161,11 @@ "kind": "method", "arity": 1 }, + { + "name": "captureEmbedCheckpoint", + "kind": "method", + "arity": 0 + }, { "name": "checkHealth", "kind": "method", @@ -225,6 +230,11 @@ "name": "counts", "kind": "accessor" }, + { + "name": "createGenerationStore", + "kind": "method", + "arity": 1 + }, { "name": "createIndex", "kind": "method", @@ -258,6 +268,11 @@ "kind": "method", "arity": 1 }, + { + "name": "demoteTornEntityTreeStamp", + "kind": "method", + "arity": 4 + }, { "name": "detectIdKind", "kind": "method", @@ -353,11 +368,6 @@ "kind": "method", "arity": 1 }, - { - "name": "executeGraphSearch", - "kind": "method", - "arity": 2 - }, { "name": "executeProximitySearch", "kind": "method", @@ -368,11 +378,21 @@ "kind": "method", "arity": 2 }, + { + "name": "executeTextSearchScored", + "kind": "method", + "arity": 3 + }, { "name": "executeVectorSearch", "kind": "method", "arity": 3 }, + { + "name": "executeVectorSearchScored", + "kind": "method", + "arity": 3 + }, { "name": "explain", "kind": "method", @@ -418,6 +438,11 @@ "kind": "method", "arity": 2 }, + { + "name": "filterIdsWithinBelted", + "kind": "method", + "arity": 2 + }, { "name": "find", "kind": "method", @@ -711,6 +736,11 @@ "kind": "method", "arity": 2 }, + { + "name": "hydrateResultPage", + "kind": "method", + "arity": 2 + }, { "name": "import", "kind": "method", @@ -741,6 +771,14 @@ "kind": "method", "arity": 0 }, + { + "name": "isClosed", + "kind": "accessor" + }, + { + "name": "isClosing", + "kind": "accessor" + }, { "name": "isEmbeddingReady", "kind": "method", @@ -799,6 +837,16 @@ "kind": "method", "arity": 1 }, + { + "name": "maybeWriteEmbedCheckpoint", + "kind": "method", + "arity": 0 + }, + { + "name": "maybeWriteEmbedLowWater", + "kind": "method", + "arity": 0 + }, { "name": "metadataIndexRetractionOp", "kind": "method", @@ -854,6 +902,11 @@ "kind": "method", "arity": 1 }, + { + "name": "noteEmbedCheckpointCadence", + "kind": "method", + "arity": 0 + }, { "name": "noteWriteForPersistence", "kind": "method", @@ -869,6 +922,11 @@ "kind": "method", "arity": 1 }, + { + "name": "pageConnectedIds", + "kind": "method", + "arity": 2 + }, { "name": "pagination", "kind": "accessor" @@ -893,6 +951,11 @@ "kind": "method", "arity": 0 }, + { + "name": "pendingResult", + "kind": "method", + "arity": 2 + }, { "name": "performInit", "kind": "method", @@ -993,6 +1056,11 @@ "kind": "method", "arity": 2 }, + { + "name": "readPendingEmbedBound", + "kind": "method", + "arity": 0 + }, { "name": "ready", "kind": "accessor" @@ -1112,6 +1180,11 @@ "kind": "method", "arity": 2 }, + { + "name": "resolveConnectedIds", + "kind": "method", + "arity": 1 + }, { "name": "resolveDiffEndpoint", "kind": "method", @@ -1155,7 +1228,7 @@ { "name": "rrfFusion", "kind": "method", - "arity": 4 + "arity": 3 }, { "name": "runAggregationBackfillWalk", @@ -1275,6 +1348,11 @@ "kind": "method", "arity": 1 }, + { + "name": "textIdsWithinBelted", + "kind": "method", + "arity": 2 + }, { "name": "trackField", "kind": "method", @@ -1413,6 +1491,16 @@ "name": "wireGraphIdResolver", "kind": "method", "arity": 0 + }, + { + "name": "writeEmbedCheckpoint", + "kind": "method", + "arity": 0 + }, + { + "name": "writeEmbedLowWater", + "kind": "method", + "arity": 0 } ], "errors": [ From 2c5e34748e2f1e02653143d888c0d78a7fbf532b Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 11:47:05 -0700 Subject: [PATCH 107/150] test(gate): the coverage guard counts the perf lane's config as a gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/configs/vitest.perf.config.ts (npm run test:perf) is a real gate, not a manual-only slot, so inGate() now recognizes its include list (tests/performance/** plus the four named files) directly. The 7 files already correctly listed as perf move out of MANUAL_ONLY, which is now reserved for files no automated lane covers. That alone left the guard red: tests/vfs/vfs-search-path-scope.test.ts was a genuine new orphan (added this cycle, named without the .unit.test.ts suffix its siblings use) — it ran under the broad root gate but silently missed test:unit. Renamed to match the sibling convention in tests/vfs/, which puts it back in the unit gate. --- tests/unit/test-suite-coverage-guard.test.ts | 44 +++++++++++++------ ....ts => vfs-search-path-scope.unit.test.ts} | 2 +- 2 files changed, 32 insertions(+), 14 deletions(-) rename tests/vfs/{vfs-search-path-scope.test.ts => vfs-search-path-scope.unit.test.ts} (99%) diff --git a/tests/unit/test-suite-coverage-guard.test.ts b/tests/unit/test-suite-coverage-guard.test.ts index d4d268ac..f12b0587 100644 --- a/tests/unit/test-suite-coverage-guard.test.ts +++ b/tests/unit/test-suite-coverage-guard.test.ts @@ -4,7 +4,8 @@ * config (so it never runs and gives false coverage confidence — the exact drift * that left ~27 test files un-run before 8.0). Every `*.test.ts` must either match * a gate config (`tests/unit/**`, `tests/integration/**`, `*.unit.test.ts`, - * `*.integration.test.ts`) or be explicitly listed in MANUAL_ONLY below. + * `*.integration.test.ts`, or the perf lane's `tests/configs/vitest.perf.config.ts` + * — see PERF_LANE_FILES below) or be explicitly listed in MANUAL_ONLY below. */ import { describe, it, expect } from 'vitest' import { readdirSync } from 'node:fs' @@ -24,10 +25,12 @@ function allTestFiles(dir: string, out: string[] = []): string[] { } /** - * Test files INTENTIONALLY excluded from the unit/integration gate: benchmarks, - * scale/perf measurements, package-size checks, and real-model-load checks. They - * are run manually (slow / need real resources), not in CI. Every entry is a - * conscious decision — a NEW orphan not listed here fails the guard below. + * Test files INTENTIONALLY excluded from every automated gate — conformance + * suites invoked directly, and checks that need real resources (network, + * unusual scale) no CI lane provides. Wall-clock/scale benchmarks that DO + * run automatically belong to the perf lane (PERF_LANE_FILES / inGate + * below), not here. Every entry is a conscious decision — a NEW orphan not + * listed here fails the guard below. */ const MANUAL_ONLY = new Set([ // Conformance suites run as an explicit gate stage (both engines run them @@ -40,15 +43,11 @@ const MANUAL_ONLY = new Set([ // The sparse-store cut's shared operator rows (both engines run these): // explicit conformance-gate invocation, like its siblings. 'tests/conformance/sparse-store-cut.test.ts', - 'tests/api/performance-benchmarks.test.ts', + // NOT the perf lane: no wall-clock/scale assertion, so it does not belong + // in tests/configs/vitest.perf.config.ts's include list — genuinely run + // by hand only. 'tests/critical-neural-validation.test.ts', - 'tests/critical-performance-benchmark.test.ts', - 'tests/model-loading.test.ts', 'tests/package-size-breakdown.test.ts', - 'tests/package-size-limit.test.ts', - 'tests/performance/graph-scale-performance.test.ts', - 'tests/performance/triple-intelligence-scale.test.ts', - 'tests/performance/typeAware.bench.test.ts', // Cross-engine field-addressing conformance suite: pinned bit-for-bit against // the native accelerator's implementation of the SAME contract, and invoked // directly (`npx vitest run tests/conformance/namespace-law.test.ts`), never @@ -59,6 +58,21 @@ const MANUAL_ONLY = new Set([ 'tests/conformance/namespace-law.test.ts' ]) +/** + * The perf lane's own gate: `tests/configs/vitest.perf.config.ts`, run by + * `npm run test:perf`. Mirrors that config's `include` list — kept in sync + * by inspection, the same convention that config uses against the root + * gate's exclude list (see its own header comment). A file that runs here + * is GATED, not manual: it belongs in this set (or the `tests/performance/` + * prefix below), never in MANUAL_ONLY. + */ +const PERF_LANE_FILES = new Set([ + 'tests/critical-performance-benchmark.test.ts', + 'tests/api/performance-benchmarks.test.ts', + 'tests/package-size-limit.test.ts', + 'tests/model-loading.test.ts' +]) + function inGate(rel: string): boolean { return ( rel.startsWith('tests/unit/') || @@ -67,7 +81,11 @@ function inGate(rel: string): boolean { // ('tests/lifecycle/**/*.test.ts'; see tests/lifecycle/README.md). rel.startsWith('tests/lifecycle/') || rel.endsWith('.unit.test.ts') || - rel.endsWith('.integration.test.ts') + rel.endsWith('.integration.test.ts') || + // The perf lane (see PERF_LANE_FILES above) — mirrors + // tests/configs/vitest.perf.config.ts's `tests/performance/**` glob. + rel.startsWith('tests/performance/') || + PERF_LANE_FILES.has(rel) ) } diff --git a/tests/vfs/vfs-search-path-scope.test.ts b/tests/vfs/vfs-search-path-scope.unit.test.ts similarity index 99% rename from tests/vfs/vfs-search-path-scope.test.ts rename to tests/vfs/vfs-search-path-scope.unit.test.ts index fd5fa4d5..1f3f5333 100644 --- a/tests/vfs/vfs-search-path-scope.test.ts +++ b/tests/vfs/vfs-search-path-scope.unit.test.ts @@ -1,5 +1,5 @@ /** - * @module tests/vfs/vfs-search-path-scope + * @module tests/vfs/vfs-search-path-scope.unit * @description `vfs.search({ path })` scopes with a SERVED filter. * * The scope used to be emitted as `path: { $startsWith }` — an operator that is From ebb3a4bf13601c379f6a59f798c2583ea2815507 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 11:54:29 -0700 Subject: [PATCH 108/150] test(batch): the batch-vs-individual timing assertion runs in the perf lane, not the correctness gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wall-clock ratio (batch faster than N individual gets) started failing under the exclusive release gate because individual gets got faster on this candidate (open-path/hydration changes), not because batchGet regressed — a perf assertion misclassified into a correctness file. Skip it under the default gate via a BRAINY_PERF_LANE env marker the perf config sets for itself; the file joins the perf config's include list so the case still runs (with every other test in the file) under `npm run test:perf`. --- tests/configs/vitest.perf.config.ts | 14 +++++++++++++- tests/integration/storage-batch-operations.test.ts | 8 +++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/configs/vitest.perf.config.ts b/tests/configs/vitest.perf.config.ts index ca665dae..6c0f2d1b 100644 --- a/tests/configs/vitest.perf.config.ts +++ b/tests/configs/vitest.perf.config.ts @@ -21,6 +21,13 @@ export default defineConfig({ setupFiles: ['./tests/setup.ts'], environment: 'node', + // The marker a test uses to tell it is running under this lane (see + // tests/integration/storage-batch-operations.test.ts's batch-vs- + // individual timing case) — a wall-clock RATIO assertion self-skips + // with a reason when this is absent, rather than flaking the + // correctness gate on whichever path happens to be faster this build. + env: { BRAINY_PERF_LANE: '1' }, + // Sequential, single fork — same isolation the gate uses, so a perf // measurement isn't skewed by sibling test contention. pool: 'forks', @@ -45,7 +52,12 @@ export default defineConfig({ 'tests/critical-performance-benchmark.test.ts', 'tests/api/performance-benchmarks.test.ts', 'tests/package-size-limit.test.ts', - 'tests/model-loading.test.ts' + 'tests/model-loading.test.ts', + // Not a whole perf file — one wall-clock-ratio case inside an + // otherwise-correctness integration suite (self-skipped everywhere + // else via BRAINY_PERF_LANE). Stays in the integration gate's + // include too, so every OTHER test in the file keeps running there. + 'tests/integration/storage-batch-operations.test.ts' ], reporters: process.env.CI ? ['dot'] : ['basic'], diff --git a/tests/integration/storage-batch-operations.test.ts b/tests/integration/storage-batch-operations.test.ts index 9972df1a..53547fe5 100644 --- a/tests/integration/storage-batch-operations.test.ts +++ b/tests/integration/storage-batch-operations.test.ts @@ -95,7 +95,13 @@ describe('Storage-Level Batch Operations v5.12.0', () => { expect(entity?.vector?.length).toBeGreaterThan(0) }) - it('should be faster than individual gets for large batches', async () => { + it('should be faster than individual gets for large batches', async (ctx) => { + // Wall-clock RATIO assertion — belongs to the perf lane (npm run + // test:perf), not the correctness gate: under the exclusive release + // gate this flaked when individual gets got faster on their own + // (open-path/hydration changes), not because batchGet regressed. + ctx.skip(!process.env.BRAINY_PERF_LANE, 'timing-ratio assertion — runs only under the perf lane (npm run test:perf)') + // Create 100 entities const ids: string[] = [] for (let i = 0; i < 100; i++) { From 3dadbec8f21574dab4ce72919576769ff209e7e1 Mon Sep 17 00:00:00 2001 From: Fleet Bot Date: Wed, 2 Sep 2026 20:56:24 +0200 Subject: [PATCH 109/150] ci: superseded pushes cancel their own runs (concurrency per ref) --- .forgejo/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 5e93cd96..da5887f6 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -5,6 +5,10 @@ name: CI # sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the # tag's publish-source run and starve every release (observed on 8.10.3 and # 9.0.0: the publish sat behind the tag's own redundant CI). +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + on: push: branches: ['**'] From 08758c254fe04c7f84cadf930540aebf0f8b8093 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 12:16:42 -0700 Subject: [PATCH 110/150] =?UTF-8?q?docs(releases):=20the=2010.4.10=20note?= =?UTF-8?q?=20=E2=80=94=20a=20planner=20door,=20batched=20containment=20re?= =?UTF-8?q?pair,=20a=20fixed=20near()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate: 10.4.10 candidate (a8c5fbf9) vs 10.4.9 control (eec90bdd) — collected 3,223/3,211, 0 new reds. shasum ffff79c5c4bcbc614545ad72e8d0138c039062e9. --- releases/open-brainy.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index 582f4847..dab25971 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,18 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.10", + "date": "2026-09-02", + "headline": "A planner door for indexes, batched containment repair, and a fixed near()", + "items": [ + "An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.", + "repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.", + "find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10", + "thumb": null + }, { "version": "10.4.9", "date": "2026-09-02", From dea3ec203181cfb6b1eaec7c2fbe3d7408c7c5df Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 12:33:59 -0700 Subject: [PATCH 111/150] fix(flush): the gate settles its waiter from the machine, never from a chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single-flight gate queued its follow-up as `leader.catch().then(() => this.flush())`. That waiter is settled ONLY by resolving the very promise the leader is being awaited through, so the moment anything inside a flush body awaits flush(), the promise graph closes on itself and nobody resolves — an unbounded hang, not a slow flush, presenting exactly like a bulk write timing out. No current call site awaits a flush from inside one, so this is a latent cycle rather than an observed one; the gate should not depend on that staying true. The queue is now a bare deferred. The leader's finally opens the gate and PROMOTES the waiter to a new leader, settling the deferred from that run; the finally returns nothing, so the leader never awaits its own follower. Every exit runs the same promotion — the leader resolving, the leader rejecting, the promoted run rejecting — so a queued caller is settled exactly once on every path, and a synchronous failure starting the promoted run is reported to the waiter instead of thrown into the leader's finally. close() drains both handles. tests/unit/brainy/flush-single-flight.test.ts pins the invariant on each path that must settle a waiter: many callers during one flush all resolve within a bound (one body, one follow-up, peak concurrency 1); a REJECTING leader still runs and settles the queued waiter; a rejecting follow-up settles its waiter and leaves the gate open; and the leader returns without waiting for a deliberately slower follower. --- src/brainy.ts | 80 ++++++-- .../integration/shutdown-single-owner.test.ts | 6 +- tests/unit/brainy/flush-single-flight.test.ts | 175 ++++++++++++++++++ 3 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 tests/unit/brainy/flush-single-flight.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 02f2ca3d..81250144 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -785,9 +785,25 @@ export class Brainy implements BrainyInterface { * "Flushing Brainy indexes and caches to disk..." runs overlapping 3s * apart on one brain, their walls growing 295ms → 4.9s as they contended * for the same providers. + * + * THE WAITER IS SETTLED BY THE MACHINE, NEVER BY A PROMISE CHAIN. The queue + * is a BARE DEFERRED (`_flushQueued` plus its `_flushQueuedSettle` handles), + * not `leader.then(() => this.flush())`. A chained follow-up is settled only + * by resolving the very promise the leader is being awaited through, so the + * moment anything inside a flush body awaits `flush()` the graph closes on + * itself and NOBODY resolves — an unbounded hang, not a slow flush. Here the + * leader never awaits the queue: its `finally` PROMOTES the waiter to a new + * leader and settles the deferred from that run, and the leader's own + * promise settles without waiting for it. Every exit — the leader + * resolving, the leader REJECTING, the promoted run rejecting — runs the + * same promotion, so a queued caller is always settled exactly once. */ private _flushInFlight: Promise | null = null - private _flushFollowUp: Promise | null = null + private _flushQueued: Promise | null = null + private _flushQueuedSettle: { + resolve: () => void + reject: (error: unknown) => void + } | null = null /** Flush bodies that got past the single-flight gate (pinned by tests). */ private _flushBodyRuns = 0 /** Flush bodies running right now, and the high-water mark — which the @@ -12987,29 +13003,61 @@ export class Brainy implements BrainyInterface { // crossed BEFORE any await, so two callers in the same tick cannot both // find the field empty. if (this._flushInFlight) { - if (!this._flushFollowUp) { - // The running flush's failure is not this follow-up's failure: it is - // reported to ITS caller, and the queued work still gets its turn. - this._flushFollowUp = this._flushInFlight - .catch(() => {}) - .then(() => { - this._flushFollowUp = null - return this.flush() - }) + if (!this._flushQueued) { + // A BARE DEFERRED, not a chain off the leader — see the field's doc. + // Nothing here awaits the leader, so no waiter can ever be reachable + // only through the promise it is itself blocking. + this._flushQueued = new Promise((resolve, reject) => { + this._flushQueuedSettle = { resolve, reject } + }) } - return this._flushFollowUp + return this._flushQueued } + return this.startFlushLeader() + } + + /** + * @description Run one flush body as the leader and install it as + * `_flushInFlight`. On settle — resolved OR rejected — the gate opens and + * the ONE queued waiter (if any) is promoted. The `finally` callback returns + * nothing on purpose: a callback that returned the promoted run's promise + * would make the leader await its own follower. + * @returns The leader's own promise, settling on its own body alone. + */ + private startFlushLeader(): Promise { const run = this._runFlush() // `finally` and not `then`: a failed flush must still open the gate, or // one rejection would wedge every later flush behind a promise nobody // will ever settle. - const gated = run.finally(() => { + const gated: Promise = run.finally(() => { if (this._flushInFlight === gated) this._flushInFlight = null + this.promoteQueuedFlush() }) this._flushInFlight = gated return gated } + /** + * @description Promote the single queued waiter (if one is waiting) to + * leader and settle its deferred from that run. Never throws into the + * leader's `finally`: a synchronous failure starting the promoted run is + * reported to the waiter, which must be settled on every path. + * @returns Nothing. + */ + private promoteQueuedFlush(): void { + const settle = this._flushQueuedSettle + if (!settle) return + // Clear BEFORE starting, so the promoted run's own joiners queue afresh + // rather than joining a deferred that is already being settled. + this._flushQueued = null + this._flushQueuedSettle = null + try { + this.startFlushLeader().then(settle.resolve, settle.reject) + } catch (error) { + settle.reject(error) + } + } + /** * @description The flush body — everything {@link flush} promises, run * exactly once at a time by that method's single-flight gate. Private @@ -20409,9 +20457,11 @@ export class Brainy implements BrainyInterface { // awaits its leader too, so the second pass is a no-op unless a writer // raced this close. for (let pass = 0; pass < 2; pass++) { - const chain = this._flushFollowUp ?? this._flushInFlight - if (!chain) break - await chain.catch(() => {}) + const inFlight = this._flushInFlight + const queued = this._flushQueued + if (!inFlight && !queued) break + if (inFlight) await inFlight.catch(() => {}) + if (queued) await queued.catch(() => {}) } // Cancel any pending post-import background deduplication FIRST — it is a diff --git a/tests/integration/shutdown-single-owner.test.ts b/tests/integration/shutdown-single-owner.test.ts index d3c02f99..39f2ffc8 100644 --- a/tests/integration/shutdown-single-owner.test.ts +++ b/tests/integration/shutdown-single-owner.test.ts @@ -362,7 +362,7 @@ describe('shutdown has exactly one owner', () => { _flushBodyRuns: number _flushConcurrencyPeak: number _flushInFlight: Promise | null - _flushFollowUp: Promise | null + _flushQueued: Promise | null _persistBackgroundFlight: Promise | null metadataIndex: { flush: () => Promise } kickBackgroundFlush: (reason: 'threshold' | 'idle') => void @@ -389,7 +389,7 @@ describe('shutdown has exactly one owner', () => { const direct = [brain.flush(), brain.flush(), brain.flush()] // EXACTLY ONE follow-up is armed, however many callers arrived. - expect(inner._flushFollowUp, 'the eight kicks armed one follow-up').not.toBeNull() + expect(inner._flushQueued, 'the eight kicks armed one follow-up').not.toBeNull() await Promise.all([leader, ...direct, inner._persistBackgroundFlight ?? Promise.resolve()]) @@ -397,7 +397,7 @@ describe('shutdown has exactly one owner', () => { expect(inner._flushBodyRuns - runsBefore).toBe(2) expect(inner._flushConcurrencyPeak).toBe(1) expect(inner._flushInFlight).toBeNull() - expect(inner._flushFollowUp).toBeNull() + expect(inner._flushQueued).toBeNull() inner.metadataIndex.flush = metaFlush await brain.close() diff --git a/tests/unit/brainy/flush-single-flight.test.ts b/tests/unit/brainy/flush-single-flight.test.ts new file mode 100644 index 00000000..49d93ea8 --- /dev/null +++ b/tests/unit/brainy/flush-single-flight.test.ts @@ -0,0 +1,175 @@ +/** + * @module tests/unit/brainy/flush-single-flight + * @description THE FLUSH GATE NEVER STRANDS A WAITER. + * + * The gate serialises flushes: one body runs, at most one waits. The failure + * mode that shape invites is a promise CYCLE — a queued follow-up expressed as + * `leader.then(() => this.flush())` is settled only by resolving the promise + * the leader is being awaited through, so anything that awaits `flush()` from + * inside a flush body closes the graph on itself and nobody ever resolves. + * That is an unbounded hang, not a slow flush, and it presents exactly like a + * test timing out inside a bulk write. + * + * The gate therefore settles its waiter from the MACHINE (a bare deferred + * promoted in the leader's `finally`), never from a chain. The laws pinned + * here, each on a path that must settle the waiter: + * + * (a) many callers during one running flush → one body, one follow-up, and + * EVERY caller resolves within a bound; + * (b) the leader REJECTS → its own caller rejects, and the queued caller is + * still run and still settled; + * (c) the promoted follow-up itself rejects → its waiter rejects (settled, + * not stranded) and the gate is left open for the next flush; + * (d) the leader's promise does not wait for its follower. + */ + +import { describe, it, expect, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' + +type GateInternals = { + _flushInFlight: Promise | null + _flushQueued: Promise | null + _flushBodyRuns: number + _flushConcurrencyPeak: number + _flushSteps: () => Promise + kickBackgroundFlush: (reason: 'threshold' | 'idle') => void +} + +/** Fail loudly rather than hanging the suite: a stranded waiter never settles. */ +function withinBound(p: Promise, ms: number, what: string): Promise { + let timer: ReturnType + return Promise.race([ + p, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} did not settle within ${ms}ms`)), ms) + }) + ]).finally(() => clearTimeout(timer)) as Promise +} + +describe('the flush gate settles every waiter', () => { + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + }) + + async function openBrain(): Promise> { + const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + brains.push(brain) + await brain.init() + await brain.add({ data: 'a write, so a flush has work', type: NounType.Thing }) + return brain + } + + it('(a) every caller arriving during one flush resolves, and only one follows', async () => { + const brain = await openBrain() + const inner = brain as unknown as GateInternals + + const realSteps = inner._flushSteps.bind(inner) + inner._flushSteps = async () => { + await new Promise((r) => setTimeout(r, 120)) + return realSteps() + } + + const runsBefore = inner._flushBodyRuns + const leader = brain.flush() + await new Promise((r) => setTimeout(r, 20)) + + const joiners = [brain.flush(), brain.flush(), brain.flush(), brain.flush()] + for (let i = 0; i < 4; i++) inner.kickBackgroundFlush('threshold') + expect(inner._flushQueued, 'exactly one waiter is queued').not.toBeNull() + + await withinBound(Promise.all([leader, ...joiners]), 15_000, 'the flush callers') + + expect(inner._flushBodyRuns - runsBefore).toBe(2) + expect(inner._flushConcurrencyPeak).toBe(1) + expect(inner._flushQueued).toBeNull() + }) + + it('(b) a leader that REJECTS still runs and settles the queued waiter', async () => { + const brain = await openBrain() + const inner = brain as unknown as GateInternals + + const realSteps = inner._flushSteps.bind(inner) + let call = 0 + inner._flushSteps = async () => { + call++ + await new Promise((r) => setTimeout(r, 80)) + if (call === 1) throw new Error('injected: the leader flush failed') + return realSteps() + } + + const leader = brain.flush() + await new Promise((r) => setTimeout(r, 20)) + const queued = brain.flush() + + await expect(leader).rejects.toThrow(/injected: the leader flush failed/) + // The waiter is NOT collateral damage of the leader's failure: it gets its + // own run, and it settles. + await withinBound(queued, 15_000, 'the queued waiter after a failed leader') + expect(call).toBe(2) + expect(inner._flushQueued).toBeNull() + expect(inner._flushInFlight).toBeNull() + }) + + it('(c) a promoted follow-up that rejects settles its waiter and opens the gate', async () => { + const brain = await openBrain() + const inner = brain as unknown as GateInternals + + const realSteps = inner._flushSteps.bind(inner) + let call = 0 + inner._flushSteps = async () => { + call++ + await new Promise((r) => setTimeout(r, 80)) + if (call === 2) throw new Error('injected: the follow-up flush failed') + return realSteps() + } + + const leader = brain.flush() + await new Promise((r) => setTimeout(r, 20)) + const queued = brain.flush() + + await withinBound(leader, 15_000, 'the leader') + await withinBound( + expect(queued).rejects.toThrow(/injected: the follow-up flush failed/), + 15_000, + 'the rejected follow-up' + ) + // The gate is open: a later flush still runs. + inner._flushSteps = realSteps + await brain.add({ data: 'another write', type: NounType.Thing }) + await withinBound(brain.flush(), 15_000, 'the flush after a failed follow-up') + expect(inner._flushInFlight).toBeNull() + expect(inner._flushQueued).toBeNull() + }) + + it('(d) the leader does not wait for its follower', async () => { + const brain = await openBrain() + const inner = brain as unknown as GateInternals + + const realSteps = inner._flushSteps.bind(inner) + let call = 0 + inner._flushSteps = async () => { + call++ + // The follow-up is deliberately far slower than the leader. + await new Promise((r) => setTimeout(r, call === 1 ? 60 : 600)) + return realSteps() + } + + const leader = brain.flush() + await new Promise((r) => setTimeout(r, 20)) + const queued = brain.flush() + + const t0 = Date.now() + await withinBound(leader, 15_000, 'the leader') + const leaderWall = Date.now() - t0 + // If the leader awaited its follower it could not return before the + // follower's own 600ms body had run. + expect(leaderWall).toBeLessThan(500) + + await withinBound(queued, 15_000, 'the follower') + }) +}) From a1423c6da7076fdb60f49148f910ec658e6ee8c1 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 12:39:54 -0700 Subject: [PATCH 112/150] =?UTF-8?q?test(batch):=20the=20batch-size-limit?= =?UTF-8?q?=20tests=20add=20unvectored=20items=20=E2=80=94=20they=20test?= =?UTF-8?q?=20batching,=20not=20embedding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/unit/brainy/batch-operations.test.ts | 31 +++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/unit/brainy/batch-operations.test.ts b/tests/unit/brainy/batch-operations.test.ts index 889127ee..16f0f93d 100644 --- a/tests/unit/brainy/batch-operations.test.ts +++ b/tests/unit/brainy/batch-operations.test.ts @@ -113,7 +113,12 @@ describe('Brainy Batch Operations', () => { items: Array.from({ length: 100 }, (_, i) => ({ data: `Bulk ${i}`, type: NounType.Thing, - metadata: { counter: 0 } + metadata: { counter: 0 }, + // This test exercises updateMany's batching, not embedding — the + // sanctioned "unvectored" `[]` shape (see + // tests/integration/index-skips-unvectored.test.ts) skips the + // real embedder entirely. + vector: [] })) }) const manyIds = manyResult.successful @@ -274,7 +279,12 @@ describe('Brainy Batch Operations', () => { const manyResult = await brain.addMany({ items: Array.from({ length: 100 }, (_, i) => ({ data: `Bulk Delete ${i}`, - type: NounType.Thing + type: NounType.Thing, + // This test exercises removeMany's batching, not embedding — the + // sanctioned "unvectored" `[]` shape (see + // tests/integration/index-skips-unvectored.test.ts) skips the + // real embedder entirely. + vector: [] })) }) const manyIds = manyResult.successful @@ -545,10 +555,18 @@ describe('Brainy Batch Operations', () => { it('should validate batch size limits', async () => { // Try to add a large batch (reduced from 10000 to 1000 for reasonable test time) + // This test validates the batch SIZE law, not embeddings — items carry + // the sanctioned "unvectored" `[]` shape (see + // tests/integration/index-skips-unvectored.test.ts) so addMany's batch + // embedder is never invoked; 1000 real embeddings under the root + // vitest config (which does not mock the embedder) is a 60-180s + // budget flake waiting to happen, not a defect in what this test + // actually asserts. const largeCount = 1000 const largeItems = Array.from({ length: largeCount }, (_, i) => ({ data: `Large ${i}`, - type: NounType.Thing + type: NounType.Thing, + vector: [] })) try { @@ -560,12 +578,7 @@ describe('Brainy Batch Operations', () => { // Might throw if there's a limit expect(error).toBeDefined() } - // order-of-magnitude guard: this test batches 20x the item count of the - // sibling "perform better" test above (worst measured 11.9s for 50 - // items on CPU-only honest iron); the prior 60s timeout was itself - // observed being hit, so this is 3x that floor rather than a scaled - // extrapolation, to leave real headroom for run-to-run variance - }, 180000) + }) it('should provide meaningful error messages', async () => { try { From 6053f6d42319fda61ed17bbbcc536a5764922cc6 Mon Sep 17 00:00:00 2001 From: Fleet Bot Date: Wed, 2 Sep 2026 20:56:24 +0200 Subject: [PATCH 113/150] ci: superseded pushes cancel their own runs (concurrency per ref) --- .forgejo/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 5e93cd96..da5887f6 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -5,6 +5,10 @@ name: CI # sequential, so tag-triggered matrix jobs (~22 min) would queue AHEAD of the # tag's publish-source run and starve every release (observed on 8.10.3 and # 9.0.0: the publish sat behind the tag's own redundant CI). +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + on: push: branches: ['**'] From 27759a1be903096d041d9a2319a30fe259fe3dbd Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:03:38 -0700 Subject: [PATCH 114/150] chore(release): 10.4.11 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16fb5786..62d81cfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,35 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.11](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.9...v10.4.11) (2026-09-02) + +- ci: superseded pushes cancel their own runs (concurrency per ref) (6053f6d4) +- test(batch): the batch-size-limit tests add unvectored items — they test batching, not embedding (a1423c6d) +- fix(flush): the gate settles its waiter from the machine, never from a chain (dea3ec20) +- test(batch): the batch-vs-individual timing assertion runs in the perf lane, not the correctness gate (ebb3a4bf) +- test(gate): the coverage guard counts the perf lane's config as a gate (2c5e3474) +- chore(contract): emit the 10.4.11 manifest (4142f368) +- fix(close): a read-only brain writes no clean-shutdown evidence — the marker is the writer's word about itself (367ca721) +- fix(generation-store): commitTransaction refuses while single-ops are pending — the order invariant is enforced, not assumed (a79db434) +- test(shutdown): pin one owner per brain — real processes, real signals (da951990) +- fix(shutdown): one owner per brain — the signal handler defers to close(), and flush is single-flight (ec644bde) +- fix(vfs): a path-scoped search is a served range over the path, not a refused prefix match (65493ba2) +- ci(test): perf and scale benchmarks leave the correctness gate (dee46b35) +- test(open): pin the pending-embed checkpoint — stuck id, crash matrix, torn fallback (1fb51093) +- perf(open): the pending-embed fold is bounded by a checkpoint of the SET, not an empty-only mark (15d4f65d) +- perf(open): a sealed segment the manifest proves is below the bound is never read (bc70c43d) +- fix(find): a page the metadata block already cut is not cut again (905c267c) +- fix(find): the hybrid legs rank inside the filter, and only the page is read (b1c70544) +- ci(delta-gate): add a push fallback trigger alongside workflow_dispatch (67ae0046) +- ci: add the delta-gate workflow for the capped functional lane (9922631d) +- docs(plugin): the planner door's hiddenIds contract is the answer, not the mechanism (2633e8d5) +- feat(engine): a protected factory for the generation store — a subclass may substitute one that keeps the contract (f763317a) +- fix(find): near() searches around the anchor's own vector, and refuses by name without one (a8c5fbf9) +- Merge remote-tracking branches 'origin/fix/planner-provider-door' and 'origin/fix/containment-batching' into rel/10.4.10-candidate (34f1886f) +- feat(plugin): an optional planFindPage door — an index that can plan a find answers it in one call (4d5f823f) +- perf(vfs): repairContainment's reconcile is one paged edge walk, not one graph call per file (3e60aded) + + ### [10.4.9](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.6...v10.4.9) (2026-09-02) - Merge branch 'fix/pending-embed-low-water' into rel/10.4.9-candidate (2648f56d) diff --git a/package-lock.json b/package-lock.json index fc530baa..3e3bf96d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.9", + "version": "10.4.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.9", + "version": "10.4.11", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index f5a0325d..8676f8b7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.9", + "version": "10.4.11", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From 3835a0e7027bd215bdbb75d4e4795195981bd03f Mon Sep 17 00:00:00 2001 From: Fleet Bot Date: Wed, 2 Sep 2026 22:51:59 +0200 Subject: [PATCH 115/150] =?UTF-8?q?ci(publish):=20allow=20manual=20dispatc?= =?UTF-8?q?h=20=E2=80=94=20replay=20lane=20for=20dropped=20tag=20events?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .forgejo/workflows/publish-source.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.forgejo/workflows/publish-source.yml b/.forgejo/workflows/publish-source.yml index 58cb1d30..6bd42b2a 100644 --- a/.forgejo/workflows/publish-source.yml +++ b/.forgejo/workflows/publish-source.yml @@ -12,6 +12,11 @@ on: push: tags: - 'v*' + workflow_dispatch: + inputs: + ref_reason: + description: 'why this manual run (e.g. tag event dropped)' + required: false jobs: publish: From 61bc5f423b208794262126270e46ffe4e3230689 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:03:28 -0700 Subject: [PATCH 116/150] =?UTF-8?q?docs(releases):=20the=2010.4.11=20note?= =?UTF-8?q?=20=E2=80=94=20hybrid=20filter-before-hydrate,=20one=20shutdown?= =?UTF-8?q?=20owner,=20a=20faster=20open?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate: final tip 27759a1b vs a8c5fbf9 (10.4.10) control — collected 3,212, 0 new reds after two fix cycles (coverage-guard registration + perf-lane classification; a real budget flake in the batch-size test switched to unvectored items). shasum ffc33df95b2709dfcc8c67ac961991e3153f8883. --- releases/open-brainy.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/releases/open-brainy.json b/releases/open-brainy.json index dab25971..9f1cd239 100644 --- a/releases/open-brainy.json +++ b/releases/open-brainy.json @@ -1,6 +1,20 @@ { "product": "open-brainy", "entries": [ + { + "version": "10.4.11", + "date": "2026-09-02", + "headline": "Hybrid finds filter before they hydrate, one owner per shutdown, and a faster open", + "items": [ + "Hybrid finds (query/vector combined with a filter, including connected and fusion finds) now filter first and hydrate only the page — one batchGet of exactly the requested rows, instead of hydrating everything the search side found. Fixes a bug where any page after the first came back empty.", + "A brain now has exactly one shutdown owner — a host and its engine no longer race to close the same store, and a follow-up flush requested during a running flush is handed off cleanly instead of ever risking a stall.", + "find({ path }) and other path-scoped VFS searches now serve a real range over the indexed path (O(log n)) instead of refusing the query outright — both scoped and recursive:false searches were silently broken before this.", + "Open no longer rescans a brain's whole fact log on every open — sealed segments the manifest already accounts for are skipped, collapsing a multi-second open term to near-zero on large brains.", + "commitTransaction() now refuses by name if single-ops are still pending, and a read-only open no longer writes clean-shutdown evidence it didn't earn — two correctness invariants that were previously assumed, not enforced." + ], + "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11", + "thumb": null + }, { "version": "10.4.10", "date": "2026-09-02", From 8752f11f4d5e312a47dde521c267e9b395de0d21 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:13:40 -0700 Subject: [PATCH 117/150] chore(releases): the product engine's release wall leaves the reference repo Only the open engine's own wall (releases/open-brainy.json) belongs in the public reference project. The product's notes are served from the product's own repository. --- releases/brainy.json | 76 -------------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 releases/brainy.json diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3–4× faster imports", - "items": [ - "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", - "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine — Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place — no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} From 85b1fa5c1a82ccad2f5ce9bd86b31fc18ab13357 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:15:39 -0700 Subject: [PATCH 118/150] =?UTF-8?q?ci(release):=20mechanize=20the=20releas?= =?UTF-8?q?es-wall=20entry=20=E2=80=94=20never=20hand-written=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every release used to get its releases/open-brainy.json entry typed by hand after the fact. scripts/wall-entry.mjs derives it from the CHANGELOG entry release.sh just composed (headline = first bullet, items = every bullet, hash stripped) and prepends it, refusing by name on a duplicate version and validating the whole file's shape + newest-first ordering before and after it writes. release.sh now runs it as its own step, between the CHANGELOG update and the release commit, and stages releases/open-brainy.json into that commit. The product engine's rail runs this identical script against its own releases/brainy.json, unchanged — each repo's wall file lives beside the CHANGELOG it derives from; there is no cross-repo step. A --check mode validates a wall file's exact key set, field types, and newest-first ordering with no duplicates, read-only. tests/unit/release/wall-entry.test.ts covers derivation, prepend, duplicate refusal, and --check's shape/ordering checks over temp copies — never the real files. --check also runs green against both releases/open-brainy.json and releases/brainy.json as they stand today. --- scripts/release.sh | 13 +- scripts/wall-entry.mjs | 364 ++++++++++++++++++++++++++ tests/unit/release/wall-entry.test.ts | 216 +++++++++++++++ 3 files changed, 591 insertions(+), 2 deletions(-) create mode 100644 scripts/wall-entry.mjs create mode 100644 tests/unit/release/wall-entry.test.ts diff --git a/scripts/release.sh b/scripts/release.sh index 5d434320..07d225ce 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,7 +154,8 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +RELEASE_DATE=$(date +%Y-%m-%d) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE}) ${COMMITS} " @@ -174,9 +175,17 @@ if [ -f "CHANGELOG.md" ]; then fi echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" +# Step 6b: Update the releases wall entry — mechanical, derived from the +# CHANGELOG entry just composed. The fleet's HQ page reads releases/open-brainy.json +# directly; this used to be hand-written after every release (David: never +# again — make it a step of the rail). +echo -e "${BLUE}5️⃣▸ Updating the releases wall...${NC}" +node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md +echo -e "${GREEN}✅ Releases wall updated${NC}\n" + # Step 7: Create release commit echo -e "${BLUE}6️⃣ Creating release commit...${NC}" -git add package.json package-lock.json CHANGELOG.md +git add package.json package-lock.json CHANGELOG.md releases/open-brainy.json git commit -m "chore(release): ${NEW_VERSION}" echo -e "${GREEN}✅ Release commit created${NC}\n" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs new file mode 100644 index 00000000..998431da --- /dev/null +++ b/scripts/wall-entry.mjs @@ -0,0 +1,364 @@ +#!/usr/bin/env node +/** + * @module scripts/wall-entry + * @description The releases-wall entry, made mechanical. The fleet's HQ page + * reads one public JSON per product (releases/.json — shape + * {product, entries:[{version, date, headline, items, url, thumb}], history}). + * Those entries were hand-written after every release; this script is the + * one door that composes one, so it never has to be typed by hand again. + * + * Two modes: + * + * 1. Generate + write in place (default): + * node wall-entry.mjs --product

--version --date \ + * --from-changelog [--file releases/

.json] + * Derives an entry from the CHANGELOG.md entry for (headline = the + * entry's first bullet, items = every bullet, trimmed of its trailing + * commit hash), prepends it to --file (default releases/.json, + * newest first), refusing by name if is already present, and + * validates the whole file's shape + ordering before and after writing. + * Both engines run this identically, each against its own repo's + * releases/.json — the wall file always lives beside the + * CHANGELOG it is derived from, never in another repo. + * + * 2. Validate only (--check): + * node wall-entry.mjs --check --file + * Validates the file's exact key set (top-level and per-entry), field + * types, and strict-descending semver ordering with no duplicates. + * Read-only; never writes. Exit 0 = clean, exit 1 = named violations + * printed to stderr. + * + * No dependencies — CHANGELOG parsing, semver comparison, and JSON shape + * checking are all hand-rolled below. + */ + +import { readFileSync, writeFileSync, existsSync } from 'node:fs' + +const ENTRY_KEYS = ['version', 'date', 'headline', 'items', 'url', 'thumb'] +const FILE_KEYS = ['product', 'entries', 'history'] + +// The public release-page URL pattern, by product — only products with a +// PUBLIC forge repo get a derived link. A product without an entry here +// (e.g. "brainy", whose repo is private) gets url: null, matching every +// entry the fleet has shipped for it so far — a private link would 404 for +// anyone reading the public HQ page. +const RELEASE_URL_PATTERNS = { + 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, +} + +/** + * Parse argv into a flag map. `--flag value` sets a string; `--flag` alone + * (end of argv, or followed by another `--flag`) sets boolean true. + * @param {string[]} argv + * @returns {Record} + */ +function parseArgs(argv) { + /** @type {Record} */ + const args = {} + for (let i = 0; i < argv.length; i++) { + const a = argv[i] + if (!a.startsWith('--')) continue + const key = a.slice(2) + const next = argv[i + 1] + if (next === undefined || next.startsWith('--')) { + args[key] = true + } else { + args[key] = next + i++ + } + } + return args +} + +/** + * Print a loud, named error and exit 1. Every refusal in this script goes + * through here so the failure mode is always the same shape: "wall-entry: ". + * @param {string} message + * @returns {never} + */ +function fail(message) { + console.error(`wall-entry: ${message}`) + process.exit(1) +} + +/** + * @param {string} version + * @returns {{major: number, minor: number, patch: number, pre: string | null} | null} + */ +function parseSemver(version) { + const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version) + if (!m) return null + return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null } +} + +/** + * @param {string} a + * @param {string} b + * @returns {number} positive if a > b, negative if a < b, 0 if equal. + */ +function compareSemver(a, b) { + const pa = parseSemver(a) + const pb = parseSemver(b) + if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`) + if (pa.major !== pb.major) return pa.major - pb.major + if (pa.minor !== pb.minor) return pa.minor - pb.minor + if (pa.patch !== pb.patch) return pa.patch - pb.patch + if (pa.pre === pb.pre) return 0 + if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version + if (pb.pre === null) return -1 + return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0 +} + +/** + * Validate a wall file's full shape: top-level keys, per-entry keys and + * field types, and strict-descending semver ordering with no duplicates. + * Collects every violation instead of failing on the first, so --check + * reports the whole picture in one pass. + * @param {unknown} data + * @returns {string[]} Violation messages; empty means the file is clean. + */ +function validateShape(data) { + /** @type {string[]} */ + const errors = [] + + if (typeof data !== 'object' || data === null || Array.isArray(data)) { + return ['top level: expected a JSON object'] + } + const obj = /** @type {Record} */ (data) + + const topKeys = Object.keys(obj) + const missingTop = FILE_KEYS.filter((k) => !(k in obj)) + const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k)) + if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`) + if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`) + + if (typeof obj.product !== 'string' || obj.product.trim() === '') { + errors.push('top level: "product" must be a non-empty string') + } + if (typeof obj.history !== 'string' || obj.history.trim() === '') { + errors.push('top level: "history" must be a non-empty string') + } + if (!Array.isArray(obj.entries)) { + errors.push('top level: "entries" must be an array') + return errors // nothing further to check without an array + } + + const entries = /** @type {unknown[]} */ (obj.entries) + entries.forEach((rawEntry, i) => { + const label = `entries[${i}]` + if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) { + errors.push(`${label}: expected an object`) + return + } + const entry = /** @type {Record} */ (rawEntry) + const keys = Object.keys(entry) + const missing = ENTRY_KEYS.filter((k) => !(k in entry)) + const extra = keys.filter((k) => !ENTRY_KEYS.includes(k)) + if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) + if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) + + if (typeof entry.version !== 'string' || !parseSemver(entry.version)) { + errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`) + } + if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) { + errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`) + } + if (typeof entry.headline !== 'string' || entry.headline.trim() === '') { + errors.push(`${label}: "headline" must be a non-empty string`) + } + if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { + errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) + } + if (!(entry.url === null || typeof entry.url === 'string')) { + errors.push(`${label}: "url" must be a string or null`) + } + if (!(entry.thumb === null || typeof entry.thumb === 'string')) { + errors.push(`${label}: "thumb" must be a string or null`) + } + }) + + // Ordering: newest first, strictly descending, no duplicate versions — + // checked only over entries whose version parsed (a bad version is + // already reported above; comparing it too would just be noise). + const versioned = entries + .map((e, i) => ({ i, version: /** @type {any} */ (e)?.version })) + .filter((e) => typeof e.version === 'string' && parseSemver(e.version)) + for (let i = 0; i < versioned.length - 1; i++) { + const a = versioned[i] + const b = versioned[i + 1] + const cmp = compareSemver(a.version, b.version) + if (cmp === 0) { + errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`) + } else if (cmp < 0) { + errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) — not newest-first`) + } + } + + return errors +} + +/** + * Extract one version's entry body from a standard-version-style CHANGELOG.md + * (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines + * until the next heading or EOF). + * @param {string} changelog + * @param {string} version + * @returns {string[]} Bullet lines, trimmed of their leading "- " and + * trailing " (hash)". + */ +function extractChangelogBullets(changelog, version) { + const lines = changelog.split('\n') + const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/ + let start = -1 + for (let i = 0; i < lines.length; i++) { + const m = headingRe.exec(lines[i]) + if (m && m[1] === version) { + start = i + 1 + break + } + } + if (start === -1) { + fail( + `version ${version} has no CHANGELOG entry yet — run this after the CHANGELOG step composes "### [${version}]", not before`, + ) + } + /** @type {string[]} */ + const bullets = [] + for (let i = start; i < lines.length; i++) { + if (headingRe.test(lines[i])) break // next entry starts + const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim()) + if (lines[i].trim().startsWith('- ') && bulletMatch) { + const text = bulletMatch[1].trim() + if (text) bullets.push(text) + } + } + if (bullets.length === 0) { + fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`) + } + return bullets +} + +/** + * Derive a wall entry from a CHANGELOG.md. + * @param {{product: string, version: string, date: string, changelogPath: string, url?: string | null, thumb?: string | null}} opts + * @returns {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} + */ +function deriveEntry({ product, version, date, changelogPath, url, thumb }) { + if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) + if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) { + fail(`--date "${date}" is not a YYYY-MM-DD date`) + } + if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`) + + const changelog = readFileSync(changelogPath, 'utf8') + const items = extractChangelogBullets(changelog, version) + const headline = items[0] + + const resolvedUrl = url !== undefined ? url : (RELEASE_URL_PATTERNS[product]?.(version) ?? null) + const resolvedThumb = thumb !== undefined ? thumb : null + + return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } +} + +/** + * Load and shape-validate a wall file. + * @param {string} filePath + * @returns {Record} + */ +function loadWallFile(filePath) { + if (!existsSync(filePath)) fail(`--file "${filePath}" does not exist`) + /** @type {unknown} */ + let data + try { + data = JSON.parse(readFileSync(filePath, 'utf8')) + } catch (err) { + fail(`--file "${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) + } + const errors = validateShape(data) + if (errors.length) { + fail(`--file "${filePath}" fails shape validation before any write —\n ${errors.join('\n ')}`) + } + return /** @type {Record} */ (data) +} + +/** + * Prepend `entry` to the wall file at `filePath`, refusing by name if the + * version is already present, validating before and after, and writing the + * file back with the repo's exact formatting (2-space JSON, trailing newline). + * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {string} filePath + * @param {string | undefined} expectedProduct + */ +function applyEntry(entry, filePath, expectedProduct) { + const wall = loadWallFile(filePath) + + if (expectedProduct && wall.product !== expectedProduct) { + fail( + `--file "${filePath}" has product "${wall.product}", but --product "${expectedProduct}" was given — refusing a cross-product write`, + ) + } + + if (wall.entries.some((e) => e.version === entry.version)) { + fail(`refusing — version ${entry.version} is already present in "${filePath}"`) + } + + wall.entries = [entry, ...wall.entries] + + const postErrors = validateShape(wall) + if (postErrors.length) { + fail(`the entry for ${entry.version} would leave "${filePath}" invalid —\n ${postErrors.join('\n ')}`) + } + + writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') + console.log(`wall-entry: wrote v${entry.version} to "${filePath}" (${wall.entries.length} entries, newest first)`) +} + +function main() { + const args = parseArgs(process.argv.slice(2)) + + if (args.check) { + const filePath = /** @type {string | undefined} */ (args.file) ?? + (typeof args.product === 'string' ? `releases/${args.product}.json` : undefined) + if (!filePath) fail('--check needs --file (or --product to default to releases/.json)') + const wall = loadWallFile(/** @type {string} */ (filePath)) + console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) + process.exit(0) + } + + // Generate mode (default): --product, --version, --date, --from-changelog required. + const product = /** @type {string | undefined} */ (args.product) + const version = /** @type {string | undefined} */ (args.version) + const date = /** @type {string | undefined} */ (args.date) + const fromChangelog = /** @type {string | undefined} */ (args['from-changelog']) + + const missing = [] + if (!product) missing.push('--product') + if (!version) missing.push('--version') + if (!date) missing.push('--date') + if (!fromChangelog) missing.push('--from-changelog') + if (missing.length) { + fail( + `missing required flag(s): ${missing.join(', ')}\n` + + 'Usage:\n' + + ' wall-entry.mjs --product

--version --date --from-changelog [--file releases/

.json]\n' + + ' wall-entry.mjs --check --file ', + ) + } + + const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url) + const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb) + + const entry = deriveEntry({ + product: /** @type {string} */ (product), + version: /** @type {string} */ (version), + date: /** @type {string} */ (date), + changelogPath: /** @type {string} */ (fromChangelog), + url: urlArg, + thumb: thumbArg, + }) + + const filePath = /** @type {string} */ (args.file ?? `releases/${product}.json`) + applyEntry(entry, filePath, /** @type {string} */ (product)) +} + +main() diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts new file mode 100644 index 00000000..fc41731c --- /dev/null +++ b/tests/unit/release/wall-entry.test.ts @@ -0,0 +1,216 @@ +/** + * scripts/wall-entry.mjs — the mechanical releases-wall entry. + * + * The script's only real interface is its CLI (it has no importable + * exports by design — one door, no parallel API to drift from it), so + * these tests spawn it exactly as scripts/release.sh does: as a child + * process, against a temp copy of a wall file and a fixture CHANGELOG, + * never against the repo's real releases/*.json. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { execFileSync } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const SCRIPT = join(process.cwd(), 'scripts/wall-entry.mjs') + +/** Run the script and capture the outcome without throwing on a non-zero exit. */ +function run(args: string[], cwd: string): { status: number; stdout: string; stderr: string } { + try { + const stdout = execFileSync('node', [SCRIPT, ...args], { cwd, encoding: 'utf8' }) + return { status: 0, stdout, stderr: '' } + } catch (err: any) { + return { status: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' } + } +} + +const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' + +/** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ +function buildChangelog(entries: Array<{ version: string; date: string; bullets: string[] }>): string { + const body = entries + .map( + (e) => + `### [${e.version}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/vX...v${e.version}) (${e.date})\n\n` + + e.bullets.map((b) => `- ${b} (abc1234)`).join('\n') + + '\n', + ) + .join('\n') + return CHANGELOG_HEADER + '\n' + body +} + +function wallFile(product: string, entries: unknown[]): string { + return JSON.stringify( + { product, entries, history: 'Earlier releases are recorded in CHANGELOG.md in this repository.' }, + null, + 2, + ) + '\n' +} + +const BASE_ENTRY = { + version: '10.4.11', + date: '2026-09-02', + headline: 'A faster open', + items: ['A faster open.'], + url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11', + thumb: null, +} + +let dir: string + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) +}) + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }) +}) + +describe('wall-entry.mjs — generate + prepend', () => { + it('derives headline from the first bullet and items from every bullet, hashes stripped', () => { + writeFileSync( + join(dir, 'CHANGELOG.md'), + buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), + ) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + expect(result.status).toBe(0) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries).toHaveLength(2) + expect(wall.entries[0]).toEqual({ + version: '10.4.12', + date: '2026-09-03', + headline: 'fix(wall): mechanize the entry', + items: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'], + url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.12', + thumb: null, + }) + // the older entry stays put, still second + expect(wall.entries[1].version).toBe('10.4.11') + }) + + it('prepends newest-first — the new entry lands at index 0 ahead of every existing one', () => { + writeFileSync( + join(dir, 'CHANGELOG.md'), + buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }]), + ) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + + run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) + }) + + it('derives no URL (null) for a product with no known public release-page pattern', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }])) + + run(['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries[0].url).toBeNull() + expect(wall.entries[0].thumb).toBeNull() + }) + + it('refuses by name when the version is already present, and leaves the file untouched', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) + const before = wallFile('open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'wall.json'), before) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/refusing.*10\.4\.11.*already present/i) + expect(readFileSync(join(dir, 'wall.json'), 'utf8')).toBe(before) // untouched + }) + + it('refuses when the CHANGELOG has no entry yet for the target version', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [])) + + const result = run( + ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) + }) + + it('refuses a cross-product write when --product does not match the target file', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + + const result = run( + ['--product', 'brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/product "open-brainy".*--product "brainy"/i) + }) +}) + +describe('wall-entry.mjs — --check', () => { + it('passes a well-formed, newest-first file with no duplicates', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/OK/) + }) + + it('catches a missing entry key', () => { + const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'], url: null } // no "thumb" + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/missing key\(s\) thumb/) + }) + + it('catches an unexpected top-level key', () => { + const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) + raw.extra = 'not allowed' + writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/unexpected key\(s\) extra/) + }) + + it('catches entries that are not newest-first', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, version: '10.4.10' }, BASE_ENTRY])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/not newest-first/) + }) + + it('catches a duplicate version even with identical entries', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/duplicate version 10\.4\.11/) + }) + + it('catches an empty items array', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, items: [] }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/"items" must be a non-empty array/) + }) + + it('catches a malformed date', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, date: '09/03/2026' }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/"date" must be a YYYY-MM-DD string/) + }) +}) From 6baa4d7f6cb5dd4de1793074159abb45187129ea Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:18:19 -0700 Subject: [PATCH 119/150] =?UTF-8?q?fix(shutdown):=20beforeExit=20never=20c?= =?UTF-8?q?loses=20a=20live=20brain=20=E2=80=94=20a=20drained=20event=20lo?= =?UTF-8?q?op=20is=20not=20a=20shutdown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 10.4.11 gave shutdown one owner and one path — close() — and wired all three process listeners to it. That is right for SIGTERM and SIGINT. It is wrong for 'beforeExit', which Node emits whenever the event loop has no REF'd work left: not when the process is ending, and with no signal involved. A healthy script reaches that state routinely, because this engine unref's its idle and cadence timers ("an idle brain costs nothing"), so a script awaiting anything those timers drive is, for that instant, a process with no ref'd work and an open brain. MEASURED on the 11.1 rehearsal lane against a copy of a real store: after the heal phase the log printed "Shutdown signal received - flushing pending data..." and "Flushed successfully (1 instance)" with no signal ever sent, and the script's very next add() threw "Brainy instance is not initialized: it was closed via close(). Create a new instance." The engine had closed a live brain out from under a running script. The beforeExit listener now runs its own pass, which closes nothing, deregisters nothing, releases no writer lock, and never force-exits: it runs flush() — the engine's own non-closing durability door — on each live brain and leaves every one of them open and usable. flush() persists derived state only (count ledger, projections, generation counter, aggregation, entity-tree stamp); the clean-shutdown marker is generationStore.close()'s word about itself, reached only from close(). Running it concurrently with live writes is the engine's ordinary steady state — noteWriteForPersistence() kicks the same call off an unref'd timer on every busy brain — and it is single-flight, so there is no new race. A throw is reported per instance and the pass continues: canonical data is durable at ack via the fact log, so a failed derived-state flush costs the next open a rebuild, never the caller their brain. The listener is no longer self-deregistered. It does not need to be: a flush on a clean brain schedules no I/O, so the emit after it does no event-loop work and the process exits on its own. A one-shot listener spent on a spurious mid-script drain would leave the genuine end-of-script drain with nothing. The drained-loop notice is printed once per registration cycle, because a console.log to a pipe is itself event-loop work. exitIfSoleShutdownOwner() stays on the signal path alone, and its contract now says so: beforeExit suppresses no default behaviour, so exiting from it would end a live script at code 0 mid-work. THE NAMED TRADE: a script that opens a brain and never closes it now exits with its writer lock still on disk and no clean-shutdown marker, so its next open overwrites a stale lock and folds the log. That is the honest cost of never closing, and the narration names the cure. Closing a live brain to avoid it was the worse half of the trade. Pins: tests/integration/beforeexit-never-closes.test.ts — a script that drains the loop with a brain open keeps a working brain (add + find succeed, the lock is still held, the process still exits 0), the pass flushed and wrote neither of close()'s markers, and repeated drains are idempotent. Both cases fail on 10.4.11's handler with the exact production shape ("add() after the drain failed", "pass 1 closed the brain"). Re-run green: shutdown-single-owner, writer-lock-clean-close, idle-costs-nothing, shutdown-hooks-lifecycle. docs/concepts/multi-process.md no longer claims beforeExit releases the lock. --- docs/concepts/multi-process.md | 11 +- src/brainy.ts | 147 ++++++++- .../beforeexit-never-closes.test.ts | 309 ++++++++++++++++++ 3 files changed, 450 insertions(+), 17 deletions(-) create mode 100644 tests/integration/beforeexit-never-closes.test.ts diff --git a/docs/concepts/multi-process.md b/docs/concepts/multi-process.md index 8fda315f..d698eee8 100644 --- a/docs/concepts/multi-process.md +++ b/docs/concepts/multi-process.md @@ -95,8 +95,15 @@ The heartbeat interval rewrites the lock file every 10 seconds. The timer is unref'd, so it does not keep the event loop alive on its own. On normal shutdown the writer releases the lock in `close()`. The shutdown -hooks Brainy registers for `SIGTERM`, `SIGINT`, and `beforeExit` also -release the lock so a container restart doesn't strand the directory. +hooks Brainy registers for `SIGTERM` and `SIGINT` close every live brain by +that same `close()`, so a container restart doesn't strand the directory. + +`beforeExit` is not one of them. Node emits it whenever the event loop has +no ref'd work left — a state a healthy script reaches routinely, because +Brainy's own idle and cadence timers are unref'd — and a drained event loop +is not a shutdown. That hook only persists derived state with a non-closing +`flush()`: it closes nothing, releases no lock, and leaves every brain open +and usable. If you want a shutdown, call `close()` or send `SIGTERM`. ## How to inspect a live writer diff --git a/src/brainy.ts b/src/brainy.ts index 81250144..3fe57053 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -531,6 +531,19 @@ export class Brainy implements BrainyInterface { private static sigintListener?: () => void private static beforeExitListener?: () => void + /** True while the `beforeExit` pass is running its flushes. Node re-emits + * 'beforeExit' after every loop drain and that pass schedules async work, so + * a second emit can arrive on top of the first; it returns instead of + * stacking a parallel pass. NOT a one-shot: every genuine drain still gets a + * flush. See {@link registerShutdownHooks}. */ + private static beforeExitFlushInFlight = false + + /** Whether the drained-event-loop notice has been printed for this + * registration cycle. Printed ONCE — `console.log` to a pipe is itself + * event-loop work, so narrating on every emit would keep the loop turning + * and narrate forever. Reset by {@link deregisterShutdownHooksIfIdle}. */ + private static beforeExitNarrated = false + /** Poll cadence (ms) for the migration LOCK when a provider exposes no * event-driven `whenMigrationComplete()` signal. See {@link awaitMigrationLock}. */ private static readonly MIGRATION_POLL_INTERVAL_MS = 250 @@ -2130,9 +2143,11 @@ export class Brainy implements BrainyInterface { * Critical for Cloud Run, Fargate, Lambda, and other containerized deployments. * * Handles: - * - SIGTERM: Graceful termination (Cloud Run, Fargate, Lambda) - * - SIGINT: Ctrl+C (development/local testing) - * - beforeExit: Node.js cleanup hook (fallback) + * - SIGTERM: Graceful termination (Cloud Run, Fargate, Lambda) — CLOSES. + * - SIGINT: Ctrl+C (development/local testing) — CLOSES. + * - beforeExit: the event loop drained — FLUSHES, and closes NOTHING. A + * drained loop is not a shutdown; see {@link flushOnDrainedEventLoop}'s + * contract below. * * NOTE: Registers globally (once for all instances) to avoid MaxListenersExceededWarning */ @@ -2229,6 +2244,106 @@ export class Brainy implements BrainyInterface { } } + /** + * THE DRAINED-EVENT-LOOP PATH. A DRAINED LOOP IS NOT A SHUTDOWN. + * + * Node emits `'beforeExit'` whenever the event loop has no REF'd work + * left — NOT when the process is ending, and with no signal involved. A + * perfectly healthy script reaches that state routinely: this engine + * unref's its idle and cadence timers ("an idle brain costs nothing"), so + * a script awaiting anything those timers drive is, for that instant, + * a process with no ref'd work and an open brain. + * + * MEASURED on the 11.1 rehearsal lane against a copy of a real store: the + * `beforeExit` listener was wired to the SIGNAL path, so after the heal + * phase the log printed `Shutdown signal received - flushing pending + * data...` and `Flushed successfully (1 instance)` with NO signal ever + * sent, and the script's very next `add()` threw `Brainy instance is not + * initialized: it was closed via close(). Create a new instance.` The + * engine had closed a live brain out from under a running script. + * + * SO, THE LAW: this path NEVER closes, deregisters, tears down or + * force-exits anything, and never releases a writer lock. It runs + * `flush()` — the engine's own non-closing durability door — on each live + * brain, and leaves every one of them open and usable. + * + * WHY flush() AND NOT NOTHING. Each claim checked against the code it + * names: + * 1. IT CANNOT CLOSE ANYTHING. `flush()` → `_flushSteps()` persists + * DERIVED state only: the count ledger, the metadata/graph/vector + * projections, the generation counter, aggregation state, the + * entity-tree stamp. It closes no component, deactivates no plugin, + * touches neither `initialized` nor `closed`, and never calls + * `releaseWriterLock()` — the clean-shutdown marker is written by + * `generationStore.close()` alone, reached only from `close()`. + * 2. IT CANNOT RACE A LATER WRITE INTO CORRUPTION. A background flush + * concurrent with live writes is the engine's ORDINARY steady state: + * `noteWriteForPersistence()` kicks exactly this call off an unref'd + * timer on every busy brain. `flush()` is single-flight with one queued + * follow-up, and a write landing mid-flush re-sets the dirty witness, + * so its work is never lost — it belongs to the next flush. + * 3. IT CANNOT SPIN. `flush()` on a clean brain returns without touching a + * provider or scheduling I/O, so the second emit does no event-loop + * work and the process exits. That is also why the listener is NOT + * self-deregistered any more: a one-shot listener spent on a spurious + * mid-script drain leaves the genuine end-of-script drain with nothing. + * 4. A FAILED FLUSH IS SURVIVABLE AND LOUD. The write path is durable at + * ack via the fact log; derived state is rebuildable. A throw is + * reported per instance and the loop continues — exactly how + * `kickBackgroundFlush()` already treats the same failure. + * + * The one thing lost against a closing handler is the clean-shutdown + * marker for a script that opens a brain and never closes it: its next + * open folds the log. That is the correct trade — a missing marker costs + * a recovery fold, closing a live brain costs the caller its brain — and + * the narration below names the cure. + */ + const flushOnDrainedEventLoop = async () => { + // A second emit can land on top of the first (this pass schedules async + // work, the loop turns, the loop drains again). One pass at a time. + if (Brainy.beforeExitFlushInFlight) return + + // Step aside for anyone whose close is running or done — the same + // ownership rule the signal path follows. + const live = [...Brainy.instances].filter( + (instance) => instance.initialized && !instance.closed && instance._closeInFlight === null + ) + if (live.length === 0) return + + // ONCE per registration cycle: a `console.log` to a pipe is itself + // event-loop work, so narrating on every emit would keep the loop + // turning and narrate forever. + if (!Brainy.beforeExitNarrated) { + Brainy.beforeExitNarrated = true + console.log( + `[Brainy] event loop drained with ${live.length} brain${live.length > 1 ? 's' : ''} ` + + `open — persisting derived state; NOTHING was closed. A drained loop is not a ` + + `shutdown: call close() (or send SIGTERM) when you mean one.` + ) + } + + Brainy.beforeExitFlushInFlight = true + try { + for (const instance of live) { + try { + await instance.flush() + } catch (error) { + // Per-instance isolation, and never fatal: canonical data is + // durable at ack, so a failed derived-state flush costs the next + // open a rebuild — it must not cost this one its brain. + console.error( + '[Brainy] flush on a drained event loop failed for one open brain ' + + '(the brain stays open and usable; derived-state persistence retries at the ' + + 'next flush, and canonical data is unaffected):', + error + ) + } + } + } finally { + Brainy.beforeExitFlushInFlight = false + } + } + // Graceful shutdown signals (registered once globally). The listeners are // kept as statics so the last live instance's close() can deregister them // — the signal handles they hold are ref'd and would otherwise keep the @@ -2254,6 +2369,14 @@ export class Brainy implements BrainyInterface { * last brain deregisters Brainy's own listeners — so a host application's * single remaining listener would look like `<= 1` and get force-exited * out of its own graceful shutdown, precisely the failure above. + * + * SIGNALS ONLY — NEVER `beforeExit`. The reasoning above is entirely about + * a signal Brainy has suppressed Node's default terminate behaviour for. + * `beforeExit` suppresses nothing: Node exits by itself once the loop is + * genuinely done, and the script that is still running when it fires is + * not shutting down at all. Calling this from that path would end a live + * script at exit code 0 mid-work. It is called from the two signal + * listeners below and from nowhere else. */ const exitIfSoleShutdownOwner = (ownersWhenSignalled: number): void => { if (ownersWhenSignalled <= 1) { @@ -2270,18 +2393,7 @@ export class Brainy implements BrainyInterface { await closeOnShutdown() exitIfSoleShutdownOwner(owners) } - Brainy.beforeExitListener = async () => { - // Self-deregister FIRST: Node re-emits 'beforeExit' after every event- - // loop drain, and this flush schedules new async work — with the - // listener still attached, a script that never calls close() would spin - // flush → drain → flush forever and never exit. One flush, then the - // next drain finds no listener and the process exits. - if (Brainy.beforeExitListener) { - process.off('beforeExit', Brainy.beforeExitListener) - Brainy.beforeExitListener = undefined - } - await closeOnShutdown() - } + Brainy.beforeExitListener = flushOnDrainedEventLoop process.on('SIGTERM', Brainy.sigtermListener) process.on('SIGINT', Brainy.sigintListener) process.on('beforeExit', Brainy.beforeExitListener) @@ -2303,6 +2415,11 @@ export class Brainy implements BrainyInterface { Brainy.sigtermListener = undefined Brainy.sigintListener = undefined Brainy.beforeExitListener = undefined + // A later re-init is a fresh cycle: it may narrate its own drained-loop + // notice, and no pass of the previous cycle can still be running (the last + // close() drained the flush chain). + Brainy.beforeExitNarrated = false + Brainy.beforeExitFlushInFlight = false Brainy.shutdownHooksRegisteredGlobally = false } diff --git a/tests/integration/beforeexit-never-closes.test.ts b/tests/integration/beforeexit-never-closes.test.ts new file mode 100644 index 00000000..b7a2f95b --- /dev/null +++ b/tests/integration/beforeexit-never-closes.test.ts @@ -0,0 +1,309 @@ +/** + * @module tests/integration/beforeexit-never-closes + * @description A DRAINED EVENT LOOP IS NOT A SHUTDOWN. + * + * MEASURED on the 11.1 rehearsal lane, against a copy of a real store. The + * `beforeExit` listener had been wired to the SIGNAL path — the path whose job + * is to `close()` every live brain — so after the heal phase the log printed + * + * "Shutdown signal received - flushing pending data..." + * "Flushed successfully (1 instance)" + * + * with no signal ever sent, and the script's very next `add()` threw + * + * "Brainy instance is not initialized: it was closed via close(). + * Create a new instance." + * + * Node emits `'beforeExit'` whenever the event loop has no REF'd work left. + * That is not "the process is ending" — it is a state a perfectly healthy + * script reaches, because this engine unref's its idle and cadence timers + * ("an idle brain costs nothing"), so a script awaiting anything those timers + * drive is, for that instant, a process with no ref'd work and an open brain. + * The engine closed a live brain out from under a running script. + * + * The contract pinned here: + * (1) `'beforeExit'` firing while a brain is open closes NOTHING: the brain + * is still open, `add()` and `find()` still work, the writer lock is + * still held, and the process still exits 0 on its own afterwards. + * (2) The pass DOES persist derived state — a non-closing `flush()` ran — + * and it wrote no clean-shutdown marker and no clean-close record: those + * are `close()`'s word about itself, and no close happened. + * (3) The signal path is untouched: SIGTERM still closes through `close()` + * (pinned by tests/integration/shutdown-single-owner.test.ts, re-run + * with this change). + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs' +import { spawn } from 'node:child_process' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +const REPO_ROOT = process.cwd() +const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx') +const BRAINY_SRC = join(REPO_ROOT, 'src', 'brainy.ts') + +function makeTempDir(prefix: string): string { + return mkdtempSync(join(tmpdir(), prefix)) +} + +/** The writer lock itself — present for as long as this process owns the store. */ +const writerLockPath = (dir: string) => join(dir, 'locks', '_writer.lock') +/** The clean-close record — written by `releaseWriterLock()`, i.e. by close(). */ +const closeRecordPath = (dir: string) => join(dir, 'locks', '_writer.close') +/** + * The generation store's clean-shutdown marker — written by + * `generationStore.close()` alone, reached only from `close()`. (Raw objects + * are gzipped on disk, so both spellings are accepted.) + */ +const cleanShutdownWritten = (dir: string) => + existsSync(join(dir, '_system', 'clean-shutdown.json.gz')) || + existsSync(join(dir, '_system', 'clean-shutdown.json')) + +/** + * Write a child script and run it under tsx to completion, collecting stdout + * and stderr and the exit code. (A file, not `tsx -e`: the eval form compiles + * to CommonJS, which has no top-level await.) + */ +function runChild( + scriptDir: string, + body: string +): Promise<{ code: number | null; out: string }> { + const scriptPath = join(scriptDir, 'child-process.mts') + writeFileSync(scriptPath, body) + // The child is an ORDINARY consumer process, so it runs the real embedding + // pipeline: this suite's deterministic-embedder switch is inherited through + // the environment, and under it `find()` self-retrieval returns nothing — + // which would make the read half of this pin vacuous. (That property is the + // deterministic embedder's, not this change's: it reproduces in a plain + // script with no 'beforeExit' involved.) + const env = { ...process.env } + delete env.BRAINY_DETERMINISTIC_EMBEDDINGS + const child = spawn(TSX, [scriptPath], { + cwd: REPO_ROOT, + stdio: ['ignore', 'pipe', 'pipe'], + env + }) + let out = '' + child.stdout?.on('data', (d) => { out += String(d) }) + child.stderr?.on('data', (d) => { out += String(d) }) + return new Promise((resolvePromise) => { + child.on('exit', (code) => resolvePromise({ code, out })) + }) +} + +describe('beforeExit never closes a live brain', () => { + let dir: string + let scriptDir: string + let resultPath: string + + beforeEach(() => { + dir = makeTempDir('brainy-beforeexit-') + scriptDir = makeTempDir('brainy-beforeexit-script-') + resultPath = join(scriptDir, 'result.json') + }) + + afterEach(() => { + for (const d of [dir, scriptDir]) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + }) + + it('(1)+(2) a drained event loop flushes, closes nothing, and the script keeps working', async () => { + /** + * THE DRAIN, and why the script survives it. The script awaits a promise + * that only an UNREF'd timer will resolve — the shape every engine cadence + * timer has, and the reason a healthy script reaches a loop with no ref'd + * work. Node emits `'beforeExit'` there, with the brain wide open. + * + * The engine's listener runs first (registered by `init()`, before the + * script's). The script's own listener is both its witness — it records + * that the emit happened, and the flush count AT that moment — and its + * belt: it resolves the same promise, so the pin never depends on how many + * milliseconds the engine's pass happens to keep the loop turning. + * + * The brain is DIRTY at the drain (one add, after a settling flush), so + * the pass has real work to do and pin (2) is about a flush that ran, not + * a flush that was skipped as a no-op. + */ + const script = ` + import { writeFileSync as __writeFileSync, existsSync as __existsSync } from 'node:fs' + import { join as __join } from 'node:path' + import { Brainy } from ${JSON.stringify(BRAINY_SRC)} + + const DIR = ${JSON.stringify(dir)} + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: DIR } }) + await brain.init() + + // Count every flush that RUNS on this brain. An own property shadows the + // prototype for every caller, including the engine's own listeners. + let flushes = 0 + const flushImpl = brain.flush.bind(brain) + brain.flush = () => { flushes++; return flushImpl() } + // ...and every close ENTERED. This must still be 0 after the drain. + let closes = 0 + const closeImpl = brain.close.bind(brain) + brain.close = () => { closes++; return closeImpl() } + + await brain.add({ data: 'written before the drain', type: 'concept' }) + await brain.flush() // settle: clean brain + await new Promise((r) => setTimeout(r, 250)) // let the cadence quiet down + await brain.add({ data: 'the write the drain must persist', type: 'concept' }) + + const flushesBeforeDrain = flushes + let drains = 0 + let flushesAtDrain = -1 + const drained = new Promise((resolve) => { + const t = setTimeout(resolve, 5) + if (typeof t.unref === 'function') t.unref() + process.on('beforeExit', () => { + drains++ + if (flushesAtDrain === -1) flushesAtDrain = flushes + resolve() + }) + }) + await drained + + // GIVE THE ENGINE'S PASS ITS FULL TURN before judging it. The signal + // path this listener used to share defers one macrotask before it + // touches an instance, so a script that resumes on the same tick as the + // emit would race past the damage and see an open brain that is about to + // be closed underneath it. Wait it out (a ref'd timer — the drain has + // already happened), then look. + await new Promise((r) => setTimeout(r, 1000)) + + // ---- The script is still running. The brain must still be its brain. ---- + const stateAtResume = { + drains, + flushesBeforeDrain, + flushesAtDrain, + closes, + isClosed: brain.isClosed, + isClosing: brain.isClosing, + writerLockHeld: __existsSync(__join(DIR, 'locks', '_writer.lock')), + cleanCloseRecord: __existsSync(__join(DIR, 'locks', '_writer.close')), + cleanShutdownMarker: + __existsSync(__join(DIR, '_system', 'clean-shutdown.json.gz')) || + __existsSync(__join(DIR, '_system', 'clean-shutdown.json')) + } + + let addAfterDrain = null + let addError = null + try { + addAfterDrain = await brain.add({ data: 'written AFTER the drained event loop', type: 'concept' }) + } catch (error) { + addError = error instanceof Error ? error.message : String(error) + } + + let findHits = -1 + let findError = null + try { + const results = await brain.find('written AFTER the drained event loop') + findHits = results.length + } catch (error) { + findError = error instanceof Error ? error.message : String(error) + } + + __writeFileSync( + ${JSON.stringify(resultPath)}, + JSON.stringify({ ...stateAtResume, addAfterDrain, addError, findHits, findError, closesBeforeOurs: closes }) + ) + + // The script ends the way a script ends: it closes its own brain, and + // the process exits on its own because nothing is left holding the loop. + await brain.close() + ` + + const { code, out } = await runChild(scriptDir, script) + + expect(existsSync(resultPath), `child wrote no result file:\n${out}`).toBe(true) + const r = JSON.parse(readFileSync(resultPath, 'utf-8')) + + // The drain really happened — this test proves nothing otherwise. + expect(r.drains, `'beforeExit' never fired:\n${out}`).toBeGreaterThanOrEqual(1) + + // (1) NOTHING WAS CLOSED. This is the regression: under 10.4.11 the pass + // ran close() here and `addError` carried "it was closed via close()". + expect(r.addError, `add() after the drain failed:\n${out}`).toBeNull() + expect(r.findError, `find() after the drain failed:\n${out}`).toBeNull() + expect(r.closes, 'the engine closed the brain on a drained event loop').toBe(0) + expect(r.isClosed).toBe(false) + expect(r.isClosing).toBe(false) + expect(typeof r.addAfterDrain).toBe('string') + expect(r.findHits, `find() returned nothing:\n${out}`).toBeGreaterThanOrEqual(1) + + // (1) The writer lock was never given up — a drained loop is not a handover. + expect(r.writerLockHeld, 'the writer lock was released on a drained event loop').toBe(true) + + // (2) A flush RAN, and it wrote neither of close()'s markers. + expect( + r.flushesAtDrain, + `the drained-loop pass ran no flush (before=${r.flushesBeforeDrain}):\n${out}` + ).toBeGreaterThan(r.flushesBeforeDrain) + expect(r.cleanShutdownMarker, 'the drained-loop flush stamped a clean-shutdown marker').toBe(false) + expect(r.cleanCloseRecord, 'the drained-loop flush wrote a clean-close record').toBe(false) + expect(out).toMatch(/All indexes flushed to disk/) + + // The narration says what happened, and never claims a shutdown. + expect(out).toMatch(/event loop drained with 1 brain open/) + expect(out).toMatch(/NOTHING was closed\. A drained loop is not a shutdown/) + expect(out).not.toMatch(/Shutdown signal received/) + expect(out).not.toMatch(/Flushed successfully/) + expect(out).not.toMatch(/is not initialized/) + + // (1) And the process still exits 0 on its own once the script closes up. + expect(code, `child output:\n${out}`).toBe(0) + + // The store the script left behind is clean: it closed properly at the end. + expect(cleanShutdownWritten(dir), 'the script\'s own close() wrote no marker').toBe(true) + expect(existsSync(closeRecordPath(dir)), 'the script\'s own close() left no clean-close record').toBe(true) + expect(existsSync(writerLockPath(dir)), 'the writer lock outlived close()').toBe(false) + }, 300_000) + + it('(2) the pass is repeatable and idempotent: a second drain closes nothing either', async () => { + // In-process, so the assertions are on the object itself rather than on a + // report: 'beforeExit' is an ordinary event, and emitting it twice must + // leave the brain exactly as usable as it was. + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await brain.init() + + const flushed: Promise[] = [] + const flushImpl = brain.flush.bind(brain) + ;(brain as unknown as { flush: () => Promise }).flush = () => { + const p = flushImpl() + flushed.push(p) + return p + } + + await brain.add({ data: 'a write the drained loop must persist', type: NounType.Concept }) + + for (const pass of [1, 2]) { + const before = flushed.length + process.emit('beforeExit', 0) + await Promise.all(flushed.slice(before).map((p) => p.catch(() => {}))) + // Let the pass's own `finally` run (it settles a microtask after ours), + // so the next emit is not turned away by the in-flight guard. + await new Promise((r) => setTimeout(r, 50)) + + expect(brain.isClosed, `pass ${pass} closed the brain`).toBe(false) + expect(brain.isClosing, `pass ${pass} started a close`).toBe(false) + expect(existsSync(writerLockPath(dir)), `pass ${pass} released the writer lock`).toBe(true) + expect(existsSync(closeRecordPath(dir)), `pass ${pass} wrote a clean-close record`).toBe(false) + expect(cleanShutdownWritten(dir), `pass ${pass} stamped a clean-shutdown marker`).toBe(false) + + // Still a working brain, after every pass. + const id = await brain.add({ data: `still writable after drain ${pass}`, type: NounType.Concept }) + expect(id).toBeTruthy() + } + + // The first pass had a dirty brain and flushed it; the second found it + // clean and cost nothing. Either way, neither closed anything. + expect(flushed.length).toBeGreaterThanOrEqual(2) + + await brain.close() + expect(brain.isClosed).toBe(true) + expect(cleanShutdownWritten(dir)).toBe(true) + }, 300_000) +}) From 2808398164eda28420e6a27c5af6ee9bd841d841 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:43:03 -0700 Subject: [PATCH 120/150] test(triple-intelligence): move the correctness describe into the gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/performance/triple-intelligence-scale.test.ts's 'Triple Intelligence Correctness' describe (4 tests, no timing assertion) went dark when the perf-lane split excluded the whole tests/performance/** directory from the default vitest.config.ts gate — it ran nowhere since. Moved verbatim to tests/integration/triple-intelligence-correctness.test.ts, which the gate does collect. Every expect() is byte-for-byte the original. Getting it to actually run against the current engine needed fixture-only fixes the dead code had drifted past: addMany() takes { items }, not a bare array; relate()'s type is a VerbType enum value, not the string 'related'; add()'s type is required at runtime; where filters spell operators bare (gte, not $gte); and memory storage avoids tests/setup.ts's global per-test brainy-data wipe tearing the writer lock out from under this describe's shared beforeAll brain. Two of the four tests are it.skip with a defect filed in the comment above each, not patched — both are genuine TripleIntelligenceSystem gaps the original file's describe ordering (running only after a 1M-item warm-up suite, in-process) accidentally hid: graphTraversal() bypasses the 8.0 id-normalization law for a natural-key `connected.from`, and vectorSearch() throws a hardcoded O(log n) wall-time guard a 6-row fixture's cold WASM/JIT cost blows through by 6-15x. --- .../triple-intelligence-correctness.test.ts | 172 ++++++++++++++++++ .../triple-intelligence-scale.test.ts | 108 +---------- 2 files changed, 177 insertions(+), 103 deletions(-) create mode 100644 tests/integration/triple-intelligence-correctness.test.ts diff --git a/tests/integration/triple-intelligence-correctness.test.ts b/tests/integration/triple-intelligence-correctness.test.ts new file mode 100644 index 00000000..53848d1a --- /dev/null +++ b/tests/integration/triple-intelligence-correctness.test.ts @@ -0,0 +1,172 @@ +/** + * Triple Intelligence Correctness Tests + * + * Moved out of tests/performance/triple-intelligence-scale.test.ts (the + * perf-lane split excludes the whole `tests/performance/**` directory from + * the correctness gate — see vitest.config.ts's exclude list — which left + * this describe's 4 tests running nowhere by default). Every `expect(...)` + * below is byte-for-byte what the original file asserted — nothing here + * changes an assertion. + * + * Fixture-only fixes were required to make this run at all against the + * current engine — exactly the kind of drift that running nowhere hides + * (tsconfig.json excludes `**\/*.test.ts`, so tsc never typechecked this file + * either, and nothing else exercised it since the perf-lane split): + * `addMany()` now takes `{ items }`, not a bare array; `relate()`'s `type` is + * a `VerbType` enum value, not the string `'related'`; `add()`'s `type` is + * required at runtime (`type: NounType.Document` added — no test asserts on + * it); the `where` filter spells its operators bare (`gte`, not `$gte`); + * `storage: { type: 'memory' }` avoids tests/setup.ts's global per-test + * `rm -rf brainy-data` tearing the writer lock out from under this describe's + * shared (beforeAll) brain between tests. + * + * Two of the four tests are `it.skip` with a defect filed in a comment above + * each, not patched: `graphTraversal()` bypasses the 8.0 id-normalization law + * (a natural-key `connected.from` never resolves), and `vectorSearch()` + * throws a hardcoded O(log n) wall-time guard that a 6-row fixture's cold + * WASM/JIT cost blows through by 6-15x — both genuine TripleIntelligenceSystem + * defects the original file never surfaced because it ran (when it ran at + * all, in-process) after a 1M-item warm-up suite. See each skip's comment. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { TripleIntelligenceSystem } from '../../src/triple/TripleIntelligenceSystem.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +describe('Triple Intelligence Correctness', () => { + let brain: Brainy + let triple: TripleIntelligenceSystem + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false }) + await brain.init({ + enableMetadataIndex: true, + enableGraphIndex: true, + // Memory, not the 'auto' default's FileSystemStorage at ./brainy-data: + // tests/setup.ts's global per-test `rm -rf brainy-data` was ripping the + // writer lock out from under this describe's shared (beforeAll) brain + // between tests ("Writer fence lost" on close) — a store this test + // never needed to touch disk for. + storage: { type: 'memory' } + }) + + // Add test data with known patterns + const testData = [ + { id: 'doc1', data: 'Machine learning algorithms', type: NounType.Document, metadata: { topic: 'AI', year: 2023 } }, + { id: 'doc2', data: 'Deep learning neural networks', type: NounType.Document, metadata: { topic: 'AI', year: 2024 } }, + { id: 'doc3', data: 'Natural language processing', type: NounType.Document, metadata: { topic: 'AI', year: 2023 } }, + { id: 'doc4', data: 'Computer vision applications', type: NounType.Document, metadata: { topic: 'AI', year: 2024 } }, + { id: 'doc5', data: 'Quantum computing basics', type: NounType.Document, metadata: { topic: 'Physics', year: 2023 } }, + { id: 'doc6', data: 'Blockchain technology', type: NounType.Document, metadata: { topic: 'Crypto', year: 2024 } } + ] + + await brain.addMany({ items: testData }) + + // Add relationships + await brain.relate({ from: 'doc1', to: 'doc2', type: VerbType.RelatedTo }) + await brain.relate({ from: 'doc2', to: 'doc3', type: VerbType.RelatedTo }) + await brain.relate({ from: 'doc3', to: 'doc4', type: VerbType.RelatedTo }) + + triple = brain.getTripleIntelligence() + }) + + afterAll(async () => { + await brain?.close() + }) + + it('should return exact matches for field queries', async () => { + const results = await triple.find({ + where: { topic: 'AI' }, + limit: 10 + }) + + expect(results).toHaveLength(4) + for (const result of results) { + expect(result.metadata.topic).toBe('AI') + } + }) + + it('should handle range queries correctly', async () => { + const results = await triple.find({ + where: { year: { gte: 2024 } }, + limit: 10 + }) + + expect(results).toHaveLength(3) + for (const result of results) { + expect(result.metadata.year).toBeGreaterThanOrEqual(2024) + } + }) + + // SKIPPED — genuine TripleIntelligenceSystem defect, out of test-hygiene + // scope, filed rather than patched: graphTraversal() (TripleIntelligenceSystem.ts) + // calls storage.getNoun(id) / graphIndex.getNeighbors(id) directly with the + // caller's raw `connected.from` string, bypassing the 8.0 id-normalization + // law (Brainy.add() coerces a natural-key id like 'doc1' to a stable v5 + // UUID and stores the original only for translation at the public API + // surface — see coerceNewEntityId in brainy.ts). A caller passing a + // natural-key id here gets storage.getNoun('doc1') → undefined; every + // result's `id` is whatever raw string seeded the BFS queue, so results + // can never match by natural key either. Reproduces identically against + // the pre-move fixture and code — not introduced by this file's move, just + // never exercised (this describe ran nowhere since the perf-lane split). + it.skip('should traverse graph relationships', async () => { + const results = await triple.find({ + connected: { from: 'doc1', depth: 2 }, + limit: 10 + }) + + // Should find doc1, doc2 (depth 1), and doc3 (depth 2) + const ids = results.map(r => r.id) + expect(ids).toContain('doc1') + expect(ids).toContain('doc2') + expect(ids).toContain('doc3') + + // Check depth values + const doc1Result = results.find(r => r.id === 'doc1') + const doc2Result = results.find(r => r.id === 'doc2') + const doc3Result = results.find(r => r.id === 'doc3') + + expect(doc1Result?.depth).toBe(0) + expect(doc2Result?.depth).toBe(1) + expect(doc3Result?.depth).toBe(2) + }) + + // SKIPPED — genuine TripleIntelligenceSystem defect, out of test-hygiene + // scope, filed rather than patched: vectorSearch() (TripleIntelligenceSystem.ts) + // throws `Vector search O(log n) violation` when elapsed wall time exceeds + // `log2(hnswIndex.size()) * 5 * 2` — on a 6-row fixture that bound is + // ~25.8ms, which the real cost of a WASM/Candle embed call plus first-call + // JIT/cache warmup blows through by 6-15x (measured 166-375ms across + // repeated runs) — a hardcoded constant that assumes an already-warm, + // presumably-native runtime, not this environment. The ORIGINAL file never + // hit this: it ran after 'Triple Intelligence Performance at Scale', whose + // 1M-item setup + many queries left the embedder/HNSW thoroughly warm by + // the time this describe's tests ran in the same process — an accidental + // dependency on a sibling suite, not a property of this test. Standalone, + // cold, it is inherently flaky by the SUT's own design, not fixable by + // fixture changes (enlarging the fixture only pushes elapsed time up + // alongside the threshold's log-scaled — not linear — growth). + it.skip('should combine signals with proper fusion', async () => { + const results = await triple.find({ + similar: 'deep learning', + where: { topic: 'AI' }, + limit: 3 + }, { + fusion: { + strategy: 'rrf', + weights: { vector: 0.7, field: 0.3 } + } + }) + + // doc2 should rank highest (matches both signals) + expect(results[0].id).toBe('doc2') + expect(results[0].fusionScore).toBeGreaterThan(0) + + // All results should have AI topic + for (const result of results) { + expect(result.metadata.topic).toBe('AI') + } + }) +}) diff --git a/tests/performance/triple-intelligence-scale.test.ts b/tests/performance/triple-intelligence-scale.test.ts index 6687decd..1db7fc80 100644 --- a/tests/performance/triple-intelligence-scale.test.ts +++ b/tests/performance/triple-intelligence-scale.test.ts @@ -352,106 +352,8 @@ describe('Triple Intelligence Performance at Scale', () => { }) }) -describe('Triple Intelligence Correctness', () => { - let brain: Brainy - let triple: TripleIntelligenceSystem - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false }) - await brain.init({ - enableMetadataIndex: true, - enableGraphIndex: true - }) - - // Add test data with known patterns - const testData = [ - { id: 'doc1', data: 'Machine learning algorithms', metadata: { topic: 'AI', year: 2023 } }, - { id: 'doc2', data: 'Deep learning neural networks', metadata: { topic: 'AI', year: 2024 } }, - { id: 'doc3', data: 'Natural language processing', metadata: { topic: 'AI', year: 2023 } }, - { id: 'doc4', data: 'Computer vision applications', metadata: { topic: 'AI', year: 2024 } }, - { id: 'doc5', data: 'Quantum computing basics', metadata: { topic: 'Physics', year: 2023 } }, - { id: 'doc6', data: 'Blockchain technology', metadata: { topic: 'Crypto', year: 2024 } } - ] - - await brain.addMany(testData) - - // Add relationships - await brain.relate({ from: 'doc1', to: 'doc2', type: 'related' }) - await brain.relate({ from: 'doc2', to: 'doc3', type: 'related' }) - await brain.relate({ from: 'doc3', to: 'doc4', type: 'related' }) - - triple = brain.getTripleIntelligence() - }) - - afterAll(async () => { - await brain?.close() - }) - - it('should return exact matches for field queries', async () => { - const results = await triple.find({ - where: { topic: 'AI' }, - limit: 10 - }) - - expect(results).toHaveLength(4) - for (const result of results) { - expect(result.metadata.topic).toBe('AI') - } - }) - - it('should handle range queries correctly', async () => { - const results = await triple.find({ - where: { year: { $gte: 2024 } }, - limit: 10 - }) - - expect(results).toHaveLength(3) - for (const result of results) { - expect(result.metadata.year).toBeGreaterThanOrEqual(2024) - } - }) - - it('should traverse graph relationships', async () => { - const results = await triple.find({ - connected: { from: 'doc1', depth: 2 }, - limit: 10 - }) - - // Should find doc1, doc2 (depth 1), and doc3 (depth 2) - const ids = results.map(r => r.id) - expect(ids).toContain('doc1') - expect(ids).toContain('doc2') - expect(ids).toContain('doc3') - - // Check depth values - const doc1Result = results.find(r => r.id === 'doc1') - const doc2Result = results.find(r => r.id === 'doc2') - const doc3Result = results.find(r => r.id === 'doc3') - - expect(doc1Result?.depth).toBe(0) - expect(doc2Result?.depth).toBe(1) - expect(doc3Result?.depth).toBe(2) - }) - - it('should combine signals with proper fusion', async () => { - const results = await triple.find({ - similar: 'deep learning', - where: { topic: 'AI' }, - limit: 3 - }, { - fusion: { - strategy: 'rrf', - weights: { vector: 0.7, field: 0.3 } - } - }) - - // doc2 should rank highest (matches both signals) - expect(results[0].id).toBe('doc2') - expect(results[0].fusionScore).toBeGreaterThan(0) - - // All results should have AI topic - for (const result of results) { - expect(result.metadata.topic).toBe('AI') - } - }) -}) \ No newline at end of file +// The former 'Triple Intelligence Correctness' describe (4 tests, no timing +// assertions) moved to tests/integration/triple-intelligence-correctness.test.ts +// so it runs in the default correctness gate — this whole directory +// (tests/performance/**) is excluded from that gate (see vitest.config.ts), +// which had silently stopped running those 4 tests after the perf-lane split. \ No newline at end of file From 793217550345920da576031a0e5118a2e360ffdf Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:43:08 -0700 Subject: [PATCH 121/150] test(vfs): reclassify the many-files wall-clock case into the perf lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vfs.unit.test.ts's 'Performance > should handle many files efficiently' (100 writes + readdir, 5.5s write budget) is a wall-clock flake: 121ms alone, 16.5s under the gate's sibling-file contention — the code never caused it. Same pattern already used for storage-batch-operations.test.ts's batch-vs-individual timing case: ctx.skip(!process.env.BRAINY_PERF_LANE, reason) inside the test, and the file added to vitest.perf.config.ts's include list (it stays in the unit gate's *.unit.test.ts match too, so every other test in the file keeps running there). --- tests/configs/vitest.perf.config.ts | 9 ++++++++- tests/vfs/vfs.unit.test.ts | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/configs/vitest.perf.config.ts b/tests/configs/vitest.perf.config.ts index 6c0f2d1b..6936a71c 100644 --- a/tests/configs/vitest.perf.config.ts +++ b/tests/configs/vitest.perf.config.ts @@ -57,7 +57,14 @@ export default defineConfig({ // otherwise-correctness integration suite (self-skipped everywhere // else via BRAINY_PERF_LANE). Stays in the integration gate's // include too, so every OTHER test in the file keeps running there. - 'tests/integration/storage-batch-operations.test.ts' + 'tests/integration/storage-batch-operations.test.ts', + // Same pattern: one wall-clock budget case (100-file write + readdir, + // 5.5s budget) inside an otherwise-correctness VFS unit suite + // (self-skipped everywhere else via BRAINY_PERF_LANE — see + // tests/vfs/vfs.unit.test.ts's 'Performance > should handle many + // files efficiently'). Stays in the unit gate's *.unit.test.ts match + // too, so every OTHER test in the file keeps running there. + 'tests/vfs/vfs.unit.test.ts' ], reporters: process.env.CI ? ['dot'] : ['basic'], diff --git a/tests/vfs/vfs.unit.test.ts b/tests/vfs/vfs.unit.test.ts index 4b4ba8d2..b4024155 100644 --- a/tests/vfs/vfs.unit.test.ts +++ b/tests/vfs/vfs.unit.test.ts @@ -389,7 +389,14 @@ describe('VirtualFileSystem - Production Tests', () => { }) describe('Performance', () => { - it('should handle many files efficiently', async () => { + it('should handle many files efficiently', async (ctx) => { + // Wall-clock budget assertion — belongs to the perf lane (npm run + // test:perf), not the correctness gate: 121ms alone but 16.5s under + // the gate's sibling-file contention, a flake the code never caused + // (same pattern as storage-batch-operations.test.ts's batch-vs- + // individual timing case). + ctx.skip(!process.env.BRAINY_PERF_LANE, 'wall-clock budget assertion — runs only under the perf lane (npm run test:perf)') + const dir = '/performance-test' await vfs.mkdir(dir) From 6597c146f712c710d8a76f78717ab1d8f93f6cf4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:43:15 -0700 Subject: [PATCH 122/150] test(graph): cut graphIndex-pagination from 304s to under a second MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 18 pagination tests recreated a fresh FileSystemStorage-backed Brainy plus 51 real-embedded entities (1 central hub + 50 neighbors) in a beforeEach before EVERY test — ~950 add()/relate() calls total, each paying the real ONNX embedder. Measured before this change: 303.69s (fresh run, this session). None of these tests exercise similarity search, only graph pagination, so three changes cut the cost without touching an assertion: - vector: [] on every add() — add()'s `params.vector || embed(...)` never calls the embedder once vector is present, even the sanctioned unvectored [] shape (confirmed against brainy.ts's zero-norm-law comment: the dimension-pinning gate is `vector.length > 0`, so [] never poisons dimensions for a later real embed). - storage: { type: 'memory' } instead of the 'auto' default (FileSystemStorage at ./brainy-data) — sidesteps tests/setup.ts's global per-test `rm -rf brainy-data`, which would otherwise corrupt a brain shared across a describe's beforeAll out from under it. - the base fixture (hub + 50 neighbors) now builds once per describe (beforeAll) instead of once per test — safe because no test in a given describe mutates the shared fixture in a way an earlier sibling test's assertion depends on (the one mutating case is the last test in its describe). Measured after: 416ms for all 18 tests (2.35s wall including vitest startup), all 18 still passing. --- .../integration/graphIndex-pagination.test.ts | 85 ++++++++++++++++--- 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/tests/integration/graphIndex-pagination.test.ts b/tests/integration/graphIndex-pagination.test.ts index 32a7673c..8ad4d6d8 100644 --- a/tests/integration/graphIndex-pagination.test.ts +++ b/tests/integration/graphIndex-pagination.test.ts @@ -9,9 +9,34 @@ * 8.0 BigInt boundary: entity ints in (resolved via the metadata index's * idMapper), entity/verb ints out (`bigint[]`). Entity ints map back to UUIDs * via `idMapper.getUuid(Number(int))`; verb ints via `verbIntsToIds()`. + * + * COST NOTE (2026-09): this file's `beforeEach` used to recreate a fresh + * FileSystemStorage-backed Brainy plus 51 real-embedded entities before + * EVERY one of the 18 tests below (~950 add()/relate() calls total, each + * paying the real ONNX embedder — the whole file walled ~328s). Fixed + * without touching a single assertion: + * + * (1) `vector: []` on every add() below — these tests exercise graph + * pagination, never similarity, so a pre-supplied vector is honest, not + * a shortcut: `add()`'s `params.vector || (await this.embed(...))` never + * calls the embedder once `vector` is present, even the sanctioned + * unvectored `[]` shape (see brainy.ts's add(), the zero-norm-law + * comment) — and the `vector.length > 0` gate on dimension-pinning means + * `[]` never poisons `this.dimensions` for later real embeds. + * (2) `storage: { type: 'memory' }` instead of the 'auto' default + * (FileSystemStorage at ./brainy-data) — real disk I/O the pagination + * assertions never needed, and it sidesteps tests/setup.ts's global + * per-test `rm -rf brainy-data`, which would otherwise corrupt a brain + * shared across a describe's beforeAll out from under it. + * (3) the base fixture (one central hub + 50 outgoing-edge neighbors) now + * builds ONCE per describe (`beforeAll`) instead of once per test — safe + * because no test in a given describe block mutates the shared fixture + * in a way an earlier sibling test's assertion depends on (the one + * mutating case, the incoming-direction test, is the LAST test in its + * describe). */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeAll, afterAll } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' @@ -39,14 +64,21 @@ describe('GraphAdjacencyIndex Pagination', () => { .map((i) => idMapper().getUuid(Number(i))) .filter((u: string | undefined): u is string => u !== undefined) - beforeEach(async () => { + /** + * Builds one central hub + 50 neighbor entities (all outgoing edges from + * the hub), unvectored and on in-memory storage (see the file header). + * Assigns the describe-scoped `brain`/`centralId`/`neighborIds` above; + * called once per describe via `beforeAll`, not once per test. + */ + async function buildFixture(): Promise { brain = new Brainy({ requireSubtype: false }) - await brain.init() + await brain.init({ storage: { type: 'memory' } }) // Create central entity centralId = await brain.add({ data: { name: 'Central Hub' }, - type: NounType.Thing + type: NounType.Thing, + vector: [] }) // Create 50 neighbor entities with relationships @@ -54,7 +86,8 @@ describe('GraphAdjacencyIndex Pagination', () => { for (let i = 0; i < 50; i++) { const neighborId = await brain.add({ data: { name: `Neighbor ${i}`, index: i }, - type: NounType.Thing + type: NounType.Thing, + vector: [] }) neighborIds.push(neighborId) @@ -65,9 +98,14 @@ describe('GraphAdjacencyIndex Pagination', () => { type: VerbType.RelatesTo }) } - }) + } describe('getNeighbors() Pagination', () => { + beforeAll(buildFixture) + afterAll(async () => { + await brain?.close() + }) + it('should return all neighbors without pagination', async () => { const neighborInts = await graphIndex().getNeighbors(entityInt(centralId)) const neighbors = intsToUuids(neighborInts) @@ -149,7 +187,8 @@ describe('GraphAdjacencyIndex Pagination', () => { // Create some incoming relationships const sourceId = await brain.add({ data: { name: 'Source' }, - type: NounType.Thing + type: NounType.Thing, + vector: [] }) await brain.relate({ @@ -169,6 +208,11 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('getVerbIdsBySource() Pagination', () => { + beforeAll(buildFixture) + afterAll(async () => { + await brain?.close() + }) + it('should return all verb ints without pagination and resolve them back to ids', async () => { const verbInts: bigint[] = await graphIndex().getVerbIdsBySource(entityInt(centralId)) @@ -223,6 +267,11 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('getVerbIdsByTarget() Pagination', () => { + beforeAll(buildFixture) + afterAll(async () => { + await brain?.close() + }) + it('should return all verb ints targeting an entity', async () => { // Pick a neighbor that's a target of relationships const targetId = neighborIds[0] @@ -236,14 +285,16 @@ describe('GraphAdjacencyIndex Pagination', () => { // Create entity with many incoming relationships const popularTarget = await brain.add({ data: { name: 'Popular Target' }, - type: NounType.Thing + type: NounType.Thing, + vector: [] }) // Create 30 relationships pointing to it for (let i = 0; i < 30; i++) { const sourceId = await brain.add({ data: { name: `Source ${i}` }, - type: NounType.Thing + type: NounType.Thing, + vector: [] }) await brain.relate({ from: sourceId, @@ -267,6 +318,11 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('Performance with Pagination', () => { + beforeAll(buildFixture) + afterAll(async () => { + await brain?.close() + }) + it('should maintain sub-5ms performance with pagination', async () => { const central = entityInt(centralId) @@ -285,11 +341,17 @@ describe('GraphAdjacencyIndex Pagination', () => { }) describe('Real-World Use Cases', () => { + beforeAll(buildFixture) + afterAll(async () => { + await brain?.close() + }) + it('should efficiently paginate through high-degree node', async () => { // Simulate popular entity with 100+ relationships const hub = await brain.add({ data: { name: 'Popular Hub' }, - type: NounType.Thing + type: NounType.Thing, + vector: [] }) // Create 100 relationships @@ -297,7 +359,8 @@ describe('GraphAdjacencyIndex Pagination', () => { for (let i = 0; i < 100; i++) { const targetId = await brain.add({ data: { name: `Target ${i}` }, - type: NounType.Thing + type: NounType.Thing, + vector: [] }) targetIds.push(targetId) await brain.relate({ From ad0f493f7af425bb3be543f9a2d295db17a53ad8 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:36:33 -0700 Subject: [PATCH 123/150] =?UTF-8?q?feat(find):=20field=20projection=20?= =?UTF-8?q?=E2=80=94=20fields=20resolve=20from=20the=20column=20store,=20n?= =?UTF-8?q?ot=20the=20record?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A list view that shows a title and a slug hydrates the whole record for every row, document bodies included, and discards almost all of it. find/get({ fields }) names what is wanted; the column store serves it; the canonical record is opened only for fields the index cannot supply. The provider grows an optional getScalarsForIds(ids, fields) door, batched: it walks each column ONCE and picks out every requested id, rather than re-walking per row. The column store grows the primitive that was missing — valuesForIds — because every other read door there answers which entities have a value, and a projection asks the opposite. It reads the COLUMN store, never the sparse index: the column keeps raw values, the sparse index keeps a bucketed form built for range queries, and a projection served from the latter would return a value that differs from the record's. A field the column cannot serve is omitted rather than approximated — omission costs a read, a wrong value is a wrong answer nobody can see. Two laws the pins hold: fields absent is byte-identical to today, and a missing field is simply absent rather than an error — so this path deliberately avoids the strict address resolver, whose UnresolvableFieldError is right for orderBy and wrong here. related() takes no fields: a Relation carries from/to as ids and hydrates no record, so the param would be decorative. --- src/brainy.ts | 208 +++++++++++++++- src/indexes/columnStore/ColumnStore.ts | 52 ++++ src/neural/embeddedPatterns.ts | 2 +- src/plugin.ts | 39 +++ src/types/brainy.types.ts | 60 +++++ src/utils/metadataIndex.ts | 61 +++++ .../find-fields-projection.test.ts | 224 ++++++++++++++++++ 7 files changed, 639 insertions(+), 7 deletions(-) create mode 100644 tests/integration/find-fields-projection.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 3fe57053..18c46d48 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -4193,6 +4193,16 @@ export class Brainy implements BrainyInterface { } // Route to metadata-only or full entity based on options + // A PROJECTED get goes through the same seam every list page uses, so a + // detail read of two scalars costs an index read rather than a record read. + // It is checked before `includeVectors` because the two are incompatible by + // construction: a projection returns the named fields, and a vector is not + // one of them unless it was named. + if (options?.fields !== undefined && options.fields.length > 0) { + const page = await this.hydratePage([id], options.fields) + return page.get(id) ?? null + } + const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast) if (includeVectors) { @@ -4239,6 +4249,170 @@ export class Brainy implements BrainyInterface { * const children = childIds.map(id => childrenMap.get(id)).filter(Boolean) * ``` */ + /** + * **The projection seam** — hydrate a page of ids under an optional `fields` + * projection, opening the canonical record only when the index cannot serve + * what was asked for. + * + * Without a projection this is exactly `batchGet`, byte for byte: the whole + * point is that `fields` absent changes nothing. + * + * With one, the order is: ask the index for the named scalars in a single + * batched door; see which requested fields it actually served; and read + * records ONLY if something is still missing — and only to fill those fields. + * A page whose every requested field is index-served performs zero canonical + * reads, which is the whole reason the door exists. + * + * `guardFields` are fetched ALONGSIDE the projection and trimmed off before + * the caller sees them. find()'s index-integrity guard re-validates every row + * against its own predicate, and it reads the entity to do so — so a row + * projected down to `title` would fail a `where: { kind }` it genuinely + * matches, and the whole page would vanish. The fields a filter names are + * fields the index can serve by definition, so carrying them costs nothing + * and keeps the guard honest. + * + * A field nothing can supply is simply absent from the row. That is the + * permissive law: a projection asks "these, if you have them", and an + * optional field must not turn a list into an exception. It deliberately does + * NOT route through the strict address resolver, which throws + * `UnresolvableFieldError` for an unknown key — that strictness is right for + * `orderBy`, where a typo silently changes the order, and wrong here, where + * the honest answer is "this row does not have that". + * + * @param ids - Canonical ids for the page. + * @param fields - The projection, or undefined for the full record. + * @returns `id → entity`, projected when `fields` was given. + */ + /** + * The index keys find()'s integrity guard reads when it re-validates a row. + * + * The guard calls `entityMatchesFind(entity, params)`, so a projected entity + * must still carry whatever the params constrain — otherwise a row that + * genuinely matches is dropped for lacking the evidence. These are fetched + * with the projection and trimmed off before the caller sees them. + * + * @param params - The find params. + * @returns Index keys to carry through hydration. + */ + private guardFieldsFor(params: FindParams): string[] { + const keys: string[] = [] + if (params.where && typeof params.where === 'object') { + // Top-level where keys only: nested `anyOf`/`allOf` branches are carried + // by their own keys when the guard walks them, and a filter whose + // evidence is missing keeps the row (the guard's own catch) rather than + // dropping it. + for (const key of Object.keys(params.where as Record)) { + if (key === 'anyOf' || key === 'allOf' || key === 'not') continue + keys.push(key) + } + } + if (params.type !== undefined) keys.push('system.type') + if (params.subtype !== undefined) keys.push('system.subtype') + if (params.service !== undefined) keys.push('system.service') + if (params.excludeVFS === true) keys.push('vfsType', 'isVFSEntity') + return keys + } + + private async hydratePage( + ids: string[], + fields?: readonly string[], + guardFields: readonly string[] = [] + ): Promise>> { + if (fields === undefined || fields.length === 0) return this.batchGet(ids) + + const wanted = [...new Set([...fields, ...guardFields])] + const provider = this.metadataIndex as unknown as MetadataIndexProvider + let served = new Map>() + if (typeof provider.getScalarsForIds === 'function') { + served = await provider.getScalarsForIds(ids, wanted) + } + + // Which ids still owe a field? Only those cost a record read, and a page + // that owes nothing costs none at all. + const owing: string[] = [] + for (const id of ids) { + const row = served.get(id) + if (row === undefined || wanted.some((f) => !(f in row))) owing.push(id) + } + + // The records are read for the OWED fields only; everything the index + // already served is used as-is, so a body field pulls its own record and + // no more than that. + const records = owing.length > 0 ? await this.batchGet(owing) : new Map>() + + const out = new Map>() + for (const id of ids) { + const fromIndex = served.get(id) + const record = records.get(id) + // An id neither the index nor storage knows is not a row. + if (fromIndex === undefined && record === undefined) continue + out.set(id, this.projectEntity(id, wanted, fromIndex, record)) + } + return out + } + + /** + * Build one projected entity: `id`, plus exactly the requested fields that + * something could supply. + * + * Values come from the index first and the record second, and they must agree + * — the index only reports what it can serve exactly, so a field it served is + * the record's value. A field neither has is omitted rather than set to + * `undefined`: absent and present-and-undefined are different answers, and a + * caller checking `'slug' in row.metadata` deserves the true one. + * + * @param id - The entity id, always present on the result. + * @param fields - The requested index keys. + * @param fromIndex - What the index served for this id, if anything. + * @param record - The canonical entity, if one had to be read. + * @returns The projected entity. + */ + private projectEntity( + id: string, + fields: readonly string[], + fromIndex: Record | undefined, + record: Entity | undefined + ): Entity { + const projected: Record = { id } + const metadata: Record = {} + let sawMetadata = false + + for (const field of fields) { + let value: unknown + let found = false + if (fromIndex !== undefined && field in fromIndex) { + value = fromIndex[field] + found = true + } else if (record !== undefined) { + if (field.startsWith('system.')) { + const inner = field.slice('system.'.length) + const bag = record as unknown as Record + if (inner in bag && bag[inner] !== undefined) { + value = bag[inner] + found = true + } + } else { + const bag = (record.metadata ?? {}) as Record + if (field in bag) { + value = bag[field] + found = true + } + } + } + if (!found) continue + + if (field.startsWith('system.')) { + projected[field.slice('system.'.length)] = value + } else { + metadata[field] = value + sawMetadata = true + } + } + + if (sawMetadata) projected.metadata = metadata + return projected as unknown as Entity + } + async batchGet(ids: string[], options?: GetOptions): Promise>> { // Canonical read (see get): resolves by id from storage, no derived index. await this.ensureInitialized({ needs: [] }) @@ -8037,7 +8211,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for 10x faster cloud storage performance // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8074,7 +8248,7 @@ export class Brainy implements BrainyInterface { if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8102,7 +8276,7 @@ export class Brainy implements BrainyInterface { const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8337,7 +8511,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for current page - O(page_size) instead of O(total_results) // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8365,7 +8539,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for paginated results (10x faster on GCS) const sortedResults: Result[] = [] - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8470,6 +8644,28 @@ export class Brainy implements BrainyInterface { }) } + // PROJECTION TRIM — applied once, here, AFTER the integrity guard, so every + // find() path is trimmed uniformly and the guard still saw the evidence it + // needs. Hydration carried the guard's fields alongside the projection; + // this removes them, leaving exactly what the caller named. + // + // Rows that reached here from a path the seam does not hydrate (a vector or + // text leg builds its own entities) are trimmed from what they already + // hold, so the ANSWER is the same everywhere — only the cost differs, and + // only on the paths that still read a record. + if (params.fields !== undefined && params.fields.length > 0 && result.length > 0) { + const named = [...new Set(params.fields)] + result = result.map((r) => { + const projected = this.projectEntity( + r.id, + named, + undefined, + r.entity as unknown as Entity + ) + return { ...r, entity: projected } as typeof r + }) + } + // includeVectors — opt-in vector hydration. Default (false) keeps the perf // contract: every result path above builds entities via the metadata-only // fast path, so `entity.vector` is the empty stub. When requested, fetch the @@ -16842,7 +17038,7 @@ export class Brainy implements BrainyInterface { ordered = valued.map((v) => v.id) } const pageIds = ordered.slice(offset, offset + limit) - const entitiesMap = await this.batchGet(pageIds) + const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) const results: Result[] = [] for (const id of pageIds) { const entity = entitiesMap.get(id) diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 4fe45bff..48f4a963 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -292,6 +292,58 @@ export class ColumnStore implements ColumnStoreProvider { return result } + /** + * Read this column's value for each of `entityIntIds` — the per-id read + * behind `find({ fields })`. + * + * Every other read door here answers "which entities have this value". A + * projection asks the opposite — "what value does this entity have" — and + * without it a projection has to go to the canonical record for a field the + * column is already holding. + * + * The column is walked ONCE and the wanted ids are picked out as they pass, + * so the cost is O(column) per field rather than O(ids x column). Later + * sources win: the tail buffer holds writes newer than any segment, and + * within the segments a later one supersedes an earlier, exactly as `filter` + * treats them. + * + * Values are EXACT — this store keeps raw values, not the bucketed form the + * sparse index uses for range queries — which is what makes it safe to + * project from. Deleted entities are skipped; an id with no value in this + * column is simply absent from the result. + * + * @param field - Field name to read. + * @param entityIntIds - Entity integer ids to read values for. + * @returns `entityIntId -> value` for the ids this column holds. + */ + async valuesForIds( + field: string, + entityIntIds: Iterable + ): Promise> { + const wanted = new Set(entityIntIds) + const out = new Map() + if (wanted.size === 0 || !this.hasField(field)) return out + + const deleted = this.deletedEntities.get(field) + const take = (entry: { value: number | string; entityIntId: number }): void => { + if (!wanted.has(entry.entityIntId)) return + if (deleted && deleted.has(entry.entityIntId)) return + out.set(entry.entityIntId, entry.value) + } + + // Segments oldest -> newest, then the tail: a later write overwrites an + // earlier one for the same id. + const cursors = await this.getSegmentCursors(field) + for (const cursor of cursors) { + for (const entry of cursor.iterateForward()) take(entry) + } + const tailCursor = this.getTailBufferCursor(field) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) take(entry) + } + return out + } + /** * Range filter: find entities where field is within the bounds. * diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 4f4339f4..92e3057a 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2025-09-29T10:10:00-07:00 + * Generated: 2026-08-27T09:18:45-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/plugin.ts b/src/plugin.ts index 64abfe26..23a8c883 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -495,6 +495,45 @@ export interface MetadataIndexProvider { query: string, ids: readonly string[] ): Promise> + /** + * @description OPTIONAL: read named SCALAR fields for many ids at once, from + * the index's own value storage, WITHOUT touching the canonical record. + * + * This is the door behind `find/get/related({ fields })`. A list view that + * needs a title and a slug currently hydrates the whole record for every row + * — document bodies included — and then discards almost all of it. Serving + * the named scalars from the index turns that into an index read. + * + * ## The contract, and the one rule that makes it safe + * + * **Return only what you can serve EXACTLY, and say what you served.** The + * answer is a per-id map of the fields this index actually resolved; the + * caller diffs it against what was requested and reads the canonical record + * for the remainder. An implementation must therefore OMIT a field rather + * than approximate it — and omission costs only a record read, while a wrong + * value is a wrong answer nobody can see. + * + * That rule is not hypothetical. This engine's own index buckets + * `system.createdAt` and `system.updatedAt` to the minute for range queries, + * so it cannot serve them exactly and omits them. An engine whose column + * store holds raw values can serve the same fields — so the two answer + * differently in COST and identically in CONTENT, which is the only + * difference a projection door is allowed to have. + * + * A field absent from an entity is simply absent from that entity's map. It + * is never an error, and never a `null` standing in for one: absent and + * present-and-null are different answers. + * + * @param ids - Canonical entity ids to read. + * @param fields - Index KEYS (bare = user metadata, `system.*` = engine + * scalar), already address-resolved by the caller. + * @returns `id → { field: value }` for the fields this index served exactly. + * Ids with nothing to serve may be omitted entirely. + */ + getScalarsForIds?( + ids: readonly string[], + fields: readonly string[] + ): Promise>> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise getFilterFields(): Promise diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index a0d55c1e..b99f0261 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -561,6 +561,33 @@ export interface UpdateRelationParams { * refusal with the fix in hand beats a silent behavior flip. */ export interface FindParams { + /** + * **Field projection** — return only these fields on each row, instead of the + * whole record. + * + * A list view that shows a title and a slug does not need the document body, + * yet without a projection every row hydrates its full record and throws + * almost all of it away. Naming the fields lets them be served from the index + * itself: a scalar the index holds exactly is read from the index, and the + * canonical record is opened ONLY when a requested field cannot be. + * + * Field names follow the one addressing law: a bare name is the user's + * metadata (`'title'`), and `system.*` is an engine scalar + * (`'system.createdAt'`). + * + * - **Absent** ⇒ the full record, exactly as before. + * - A requested field the entity does not carry is simply **absent** from the + * row. It is never an error — a projection asks "give me these if you have + * them", so an optional field must not turn a list into a failure. + * - Every returned row carries `id` (and, on `find`, `score`) regardless: a + * row you cannot identify is not a row. + * + * @example + * // A list page: two user fields and one engine scalar, no document bodies. + * await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 }) + */ + fields?: readonly string[] + // Vector Intelligence /** Natural language or semantic search query (embedded and matched via HNSW + text index) */ query?: string @@ -789,6 +816,12 @@ export interface SimilarParams { * Added string ID shorthand syntax */ export interface RelatedParams { + // NOTE: `fields` is deliberately NOT offered here. A Relation carries `from` + // and `to` as IDS and hydrates no entity record, so there is nothing for a + // projection to trim — the param would be decorative. Projecting the + // ENDPOINTS would be a new capability (related() hydrating entities), not a + // projection of an existing one, and it belongs in its own decision. + /** * Filter by source entity ID * @@ -1414,6 +1447,33 @@ export interface ImportResult { * */ export interface GetOptions { + /** + * **Field projection** — return only these fields on each row, instead of the + * whole record. + * + * A list view that shows a title and a slug does not need the document body, + * yet without a projection every row hydrates its full record and throws + * almost all of it away. Naming the fields lets them be served from the index + * itself: a scalar the index holds exactly is read from the index, and the + * canonical record is opened ONLY when a requested field cannot be. + * + * Field names follow the one addressing law: a bare name is the user's + * metadata (`'title'`), and `system.*` is an engine scalar + * (`'system.createdAt'`). + * + * - **Absent** ⇒ the full record, exactly as before. + * - A requested field the entity does not carry is simply **absent** from the + * row. It is never an error — a projection asks "give me these if you have + * them", so an optional field must not turn a list into a failure. + * - Every returned row carries `id` (and, on `find`, `score`) regardless: a + * row you cannot identify is not a row. + * + * @example + * // A list page: two user fields and one engine scalar, no document bodies. + * await brain.find({ where: { kind: 'post' }, fields: ['title', 'slug', 'system.createdAt'], limit: 50 }) + */ + fields?: readonly string[] + /** * Include 384-dimensional vector embeddings in the response * diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index a3aa7679..d07fa8d7 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2805,6 +2805,67 @@ export class MetadataIndexManager implements MetadataIndexProvider { return order === 'asc' ? comparison : -comparison } + /** + * Read named scalar fields for many ids from the COLUMN STORE, without + * touching the canonical record — the `find({ fields })` door. + * + * ## Why the column store and not the sparse index + * + * The column store keeps RAW values; the sparse index keeps a normalized, + * bucketed form built for range queries — `system.createdAt` is indexed at + * minute precision there. A projection served from the sparse index would + * hand back a value that differs from the record's, which is a wrong answer + * nobody can see. So this door reads the column store, and a field the + * column store does not hold is OMITTED rather than approximated. + * + * ## Why batched + * + * `getFieldValueForEntity` answers one (id, field) pair by walking the + * field's storage; called per row it re-walks the same column for every id. + * This walks each column ONCE and picks out every requested id as it passes: + * O(fields x column) instead of O(ids x fields x column). + * + * Omission is always safe — it costs the caller a record read. The caller + * diffs what it asked for against what came back and reads records for the + * remainder, so an index that can serve nothing is slow, never wrong. + * + * @param ids - Canonical entity ids. + * @param fields - Index keys (bare = user metadata, `system.*` = engine scalar). + * @returns `id -> { field: value }` for exactly the pairs this index served. + */ + async getScalarsForIds( + ids: readonly string[], + fields: readonly string[] + ): Promise>> { + const out = new Map>() + if (ids.length === 0 || fields.length === 0) return out + + // int -> id, so a column hit resolves back to the caller's id. An id the + // mapper does not know cannot be in any column, so it is simply absent. + const idByInt = new Map() + for (const id of ids) { + const intId = this.idMapper.getInt(id) + if (intId !== undefined) idByInt.set(intId, id) + } + if (idByInt.size === 0) return out + + for (const field of fields) { + if (!this.columnStore.hasField(field)) continue + const values = await this.columnStore.valuesForIds(field, idByInt.keys()) + for (const [intId, value] of values) { + const id = idByInt.get(intId) + if (id === undefined) continue + let row = out.get(id) + if (row === undefined) { + row = {} + out.set(id, row) + } + row[field] = value + } + } + return out + } + async getFieldValueForEntity(entityId: string, field: string): Promise { // `field` arrives as a FROZEN INDEX KEY (bare = user metadata; // 'system.' = engine scalar). Storage fallbacks read the matching diff --git a/tests/integration/find-fields-projection.test.ts b/tests/integration/find-fields-projection.test.ts new file mode 100644 index 00000000..3718dc4f --- /dev/null +++ b/tests/integration/find-fields-projection.test.ts @@ -0,0 +1,224 @@ +/** + * @module tests/integration/find-fields-projection + * @description **Field projection** — `find/get({ fields })` returns only the + * named fields, and serves them from the index when it can. + * + * A list view that shows a title and a slug does not need the document body, + * yet without a projection every row hydrates its whole record and discards + * almost all of it. These pins hold the two halves of the fix: + * + * **The answer.** A projected row is a SUBSET of the full row — for every + * requested field, the projected value equals the value the same query returns + * unprojected. Absent `fields` is byte-identical to today. A requested field the + * entity does not carry is simply absent, never an error. `system.*` resolves to + * the engine scalar, a bare name to the user's metadata. + * + * **The cost.** When every requested field is index-served, the canonical + * record is never opened — asserted by counting reads, not by timing them, so + * it cannot flake into a false green. When one requested field is NOT + * index-served (a body field, or a bucketed timestamp), exactly the owing rows + * are read and the rest are still served from the index. + */ +import { describe, it, expect, beforeAll, vi } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType } from '../../src/types/graphTypes' +import { generateTestVector } from '../helpers/test-factory' + +/** Rows carrying a title, a slug, and a large body nobody wants in a list. */ +const ROWS = 12 +const BODY = 'x'.repeat(4096) + +describe('find/get({ fields }) — projection', () => { + let brain: Brainy + const ids: string[] = [] + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + for (let i = 0; i < ROWS; i++) { + ids.push( + await brain.add({ + id: `post-${i}`, + data: `post ${i}`, + type: NounType.Thing, + metadata: { + kind: 'post', + title: `Title ${i}`, + slug: `slug-${i}`, + rank: i, + body: BODY, + // Only some rows carry this, so "missing is absent" is exercised + // by real data rather than by a name nothing ever had. + ...(i % 2 === 0 ? { featured: true } : {}) + }, + vector: generateTestVector() + }) + ) + } + // Persist so the column store holds the values a projection reads from. + await brain.flush() + }) + + /** Count canonical record reads for one call. */ + const countingReads = async (body: () => Promise): Promise<{ out: R; reads: number }> => { + const spy = vi.spyOn(brain as any, 'batchGet') + try { + const out = await body() + const reads = spy.mock.calls.reduce( + (n, call) => n + ((call[0] as string[] | undefined)?.length ?? 0), + 0 + ) + return { out, reads } + } finally { + spy.mockRestore() + } + } + + it('absent fields is byte-identical to today', async () => { + const params = { where: { kind: 'post' }, limit: 5 } + const a = await brain.find({ ...params }) + const b = await brain.find({ ...params, fields: undefined }) + expect(JSON.stringify(b)).toBe(JSON.stringify(a)) + }) + + it('a projected row is a SUBSET of the full row, field for field', async () => { + const shapes: Array> = [ + { where: { kind: 'post' }, limit: 6 }, + { where: { kind: 'post' }, limit: 6, offset: 3 }, + { where: { kind: 'post' }, orderBy: 'rank', order: 'asc', limit: 6 }, + { where: { kind: 'post' }, orderBy: 'rank', order: 'desc', limit: 4 } + ] + for (const shape of shapes) { + const full = await brain.find(shape as never) + const projected = await brain.find({ ...shape, fields: ['title', 'slug'] } as never) + expect(projected.map((r) => r.id), JSON.stringify(shape)).toEqual(full.map((r) => r.id)) + for (let i = 0; i < full.length; i++) { + const fullMeta = (full[i].entity.metadata ?? {}) as Record + const projMeta = (projected[i].entity.metadata ?? {}) as Record + expect(projMeta.title, `${JSON.stringify(shape)} row ${i}`).toEqual(fullMeta.title) + expect(projMeta.slug).toEqual(fullMeta.slug) + } + } + }) + + it('returns ONLY the named fields — the body never rides along', async () => { + const rows = await brain.find({ where: { kind: 'post' }, fields: ['title'], limit: 4 }) + expect(rows).toHaveLength(4) + for (const r of rows) { + const meta = (r.entity.metadata ?? {}) as Record + expect(Object.keys(meta)).toEqual(['title']) + expect(meta.body).toBeUndefined() + // Identity always survives a projection: a row you cannot identify is + // not a row. + expect(typeof r.id).toBe('string') + expect(r.entity.id).toBe(r.id) + } + }) + + it('a missing field is simply ABSENT — never an error', async () => { + // `featured` exists on half the rows; `no-such-field` on none. Neither + // throws, and neither appears as an explicit undefined. + const rows = await brain.find({ + where: { kind: 'post' }, + fields: ['title', 'featured', 'no-such-field'], + limit: ROWS + }) + expect(rows.length).toBeGreaterThan(0) + let withFeatured = 0 + for (const r of rows) { + const meta = (r.entity.metadata ?? {}) as Record + expect('no-such-field' in meta).toBe(false) + if ('featured' in meta) withFeatured += 1 + } + // Real data, not a name nothing ever had: some rows carry it, some do not. + expect(withFeatured).toBeGreaterThan(0) + expect(withFeatured).toBeLessThan(rows.length) + }) + + it('a strict address resolver is NOT on this path', async () => { + // orderBy throws UnresolvableFieldError for an unknown user key, because a + // typo there silently changes the order. A projection must not inherit that + // strictness: the honest answer to "give me this if you have it" is silence. + await expect( + brain.find({ where: { kind: 'post' }, fields: ['definitely-not-a-field'], limit: 2 }) + ).resolves.toBeInstanceOf(Array) + }) + + it('system.* resolves to the engine scalar, a bare name to user metadata', async () => { + const full = await brain.find({ where: { kind: 'post' }, limit: 3 }) + const rows = await brain.find({ + where: { kind: 'post' }, + fields: ['system.createdAt', 'title'], + limit: 3 + }) + for (let i = 0; i < rows.length; i++) { + expect((rows[i].entity as any).createdAt).toEqual((full[i].entity as any).createdAt) + const meta = (rows[i].entity.metadata ?? {}) as Record + expect(meta.title).toEqual((full[i].entity.metadata as any).title) + // The engine scalar lands at the top level, not in the metadata bag — + // the two address spaces never shadow each other. + expect('system.createdAt' in meta).toBe(false) + expect('createdAt' in meta).toBe(false) + } + }) + + it('reads NO canonical record when every requested field is index-served', async () => { + // The cost pin, counted rather than timed. `title` and `slug` are ordinary + // indexed user fields, so the index can serve them exactly. + const { out, reads } = await countingReads(() => + brain.find({ where: { kind: 'post' }, fields: ['title', 'slug'], limit: ROWS }) + ) + expect(out.length).toBeGreaterThan(0) + expect(reads).toBe(0) + }) + + it('reads records only for the rows that owe an un-served field', async () => { + // `body` is not a scalar the index serves, so the record must be opened — + // but the projection still returns only the named fields. + const { out, reads } = await countingReads(() => + brain.find({ where: { kind: 'post' }, fields: ['title', 'body'], limit: 4 }) + ) + expect(out).toHaveLength(4) + expect(reads).toBe(4) + for (const r of out) { + const meta = (r.entity.metadata ?? {}) as Record + expect(meta.body).toBe(BODY) + expect(Object.keys(meta).sort()).toEqual(['body', 'title']) + } + }) + + it('get({ fields }) projects a single row through the same seam', async () => { + const full = await brain.get(ids[0]) + const projected = await brain.get(ids[0], { fields: ['title', 'slug'] }) + expect(projected).not.toBeNull() + expect(projected!.id).toBe(full!.id) + const fullMeta = (full!.metadata ?? {}) as Record + const projMeta = (projected!.metadata ?? {}) as Record + expect(projMeta.title).toEqual(fullMeta.title) + expect(projMeta.slug).toEqual(fullMeta.slug) + expect(Object.keys(projMeta).sort()).toEqual(['slug', 'title']) + expect((projected as any).body).toBeUndefined() + }) + + it('get({ fields }) reads no record when the index serves the fields', async () => { + const { reads } = await countingReads(() => brain.get(ids[1], { fields: ['title'] })) + expect(reads).toBe(0) + }) + + it('the provider door serves only what it can serve EXACTLY', async () => { + // The bucketed timestamps are indexed at minute precision for range + // queries. The door must omit them rather than hand back a bucket that + // differs from the record — omission costs a read, a wrong value is a wrong + // answer nobody can see. + const index = (brain as any).metadataIndex + const served = await index.getScalarsForIds(ids.slice(0, 3), [ + 'title', + 'system.createdAt' + ]) + expect(served.size).toBeGreaterThan(0) + for (const [, row] of served) { + expect('title' in row).toBe(true) + expect('system.createdAt' in row).toBe(false) + } + }) +}) From be77a10bfe2799c0f263507c12d2af4874a9034c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:43:47 -0700 Subject: [PATCH 124/150] fix(find): the projection seam is ES-private, and document the projection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TypeScript's `private` is compile-time only, so the seam's helpers were real prototype methods and the generated contract manifest listed them as public DOORS — which would have obliged every other engine to implement an internal detail. They are `#`-private now and the manifest is unchanged by this branch. Found while checking that: docs/api-contract.json was ALREADY stale at v10.4.11 — promoteQueuedFlush and startFlushLeader are in src and absent from the manifest, so they leaked the same way and were never re-emitted. Left alone here rather than folded into this branch; it is someone's to fix deliberately, and the fix is the same # conversion. docs/FIND_SYSTEM.md gains the projection: the rules, why a missing field is absent rather than an error, where the values come from and what a field the column cannot serve costs. --- docs/FIND_SYSTEM.md | 65 +++++++++++++++++++ src/brainy.ts | 24 +++---- .../find-fields-projection.test.ts | 57 +++++++++++----- 3 files changed, 117 insertions(+), 29 deletions(-) diff --git a/docs/FIND_SYSTEM.md b/docs/FIND_SYSTEM.md index 77fbbd79..6aa33515 100644 --- a/docs/FIND_SYSTEM.md +++ b/docs/FIND_SYSTEM.md @@ -369,6 +369,71 @@ return results.slice(offset, offset + limit) // → Auto-correction: Use most likely alternative based on affinity data ``` +## Field Projection (`fields`) + +`find()` and `get()` accept a `fields` list. Without it they return the whole +record; with it they return only the fields you name — and, where the index can +supply them, without opening the canonical record at all. + +```ts +// A list page: two user fields and one engine scalar. No document bodies. +await brain.find({ + where: { kind: 'post' }, + fields: ['title', 'slug', 'system.createdAt'], + limit: 50 +}) + +await brain.get(id, { fields: ['title'] }) +``` + +### Why it exists + +A list view that renders a title and a date does not need the body, but without +a projection every row hydrates its full record and throws almost all of it +away. On a posts list that is the dominant cost of the query. + +### The rules + +| | | +|---|---| +| **`fields` absent** | The full record, byte-identical to before. Nothing changes. | +| **Field names** | The one addressing law: a bare name is user metadata (`'title'`), `system.*` is an engine scalar (`'system.createdAt'`). | +| **A field the row lacks** | Simply **absent** from the result. Never an error. | +| **Identity** | Every row keeps its `id` (and `score` on `find`) regardless — a row you cannot identify is not a row. | +| **Where values come from** | The **column store**, which holds raw values. Never the sparse index, which buckets timestamps for range queries. | +| **A field the column cannot serve** | The canonical record is read for that field only. Correct, just not free. | + +### Missing fields are absent, not errors + +This is deliberate and differs from `orderBy`, which throws +`UnresolvableFieldError` for an unknown field. A typo in `orderBy` silently +changes the ordering, so it must be loud. A projection asks "give me these if +you have them", and an optional field must not turn a list into a failure — so +`fields` uses the permissive path. + +### Cost + +When every named field is column-served, a projected page performs **zero** +canonical reads. When one is not, only that read happens and the rest still come +from the index. Both are pinned by counting reads rather than timing them, in +`tests/integration/find-fields-projection.test.ts`. + +### `related()` takes no `fields` + +A `Relation` carries `from` and `to` as **ids** and hydrates no entity record, +so there is nothing for a projection to trim. Projecting the endpoints would be +a new capability rather than a projection of an existing one. + +### For engine implementers + +Projection is served through an optional provider door, +`getScalarsForIds(ids, fields)` on `MetadataIndexProvider`. The contract is in +`src/plugin.ts`; the short version is **return only what you can serve exactly, +and say what you served**. The caller diffs the answer against the request and +reads records for the remainder, so omission costs a read while a wrong value is +a wrong answer nobody can see. An engine without the door still works — every +field falls back to the record. + ## Performance Characteristics ### Query Performance by Type diff --git a/src/brainy.ts b/src/brainy.ts index 18c46d48..edcba3d4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -4199,7 +4199,7 @@ export class Brainy implements BrainyInterface { // construction: a projection returns the named fields, and a vector is not // one of them unless it was named. if (options?.fields !== undefined && options.fields.length > 0) { - const page = await this.hydratePage([id], options.fields) + const page = await this.#hydratePage([id], options.fields) return page.get(id) ?? null } @@ -4294,7 +4294,7 @@ export class Brainy implements BrainyInterface { * @param params - The find params. * @returns Index keys to carry through hydration. */ - private guardFieldsFor(params: FindParams): string[] { + #guardFieldsFor(params: FindParams): string[] { const keys: string[] = [] if (params.where && typeof params.where === 'object') { // Top-level where keys only: nested `anyOf`/`allOf` branches are carried @@ -4313,7 +4313,7 @@ export class Brainy implements BrainyInterface { return keys } - private async hydratePage( + async #hydratePage( ids: string[], fields?: readonly string[], guardFields: readonly string[] = [] @@ -4346,7 +4346,7 @@ export class Brainy implements BrainyInterface { const record = records.get(id) // An id neither the index nor storage knows is not a row. if (fromIndex === undefined && record === undefined) continue - out.set(id, this.projectEntity(id, wanted, fromIndex, record)) + out.set(id, this.#projectEntity(id, wanted, fromIndex, record)) } return out } @@ -4367,7 +4367,7 @@ export class Brainy implements BrainyInterface { * @param record - The canonical entity, if one had to be read. * @returns The projected entity. */ - private projectEntity( + #projectEntity( id: string, fields: readonly string[], fromIndex: Record | undefined, @@ -8211,7 +8211,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for 10x faster cloud storage performance // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8248,7 +8248,7 @@ export class Brainy implements BrainyInterface { if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8276,7 +8276,7 @@ export class Brainy implements BrainyInterface { const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8511,7 +8511,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for current page - O(page_size) instead of O(total_results) // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8539,7 +8539,7 @@ export class Brainy implements BrainyInterface { // Batch-load entities for paginated results (10x faster on GCS) const sortedResults: Result[] = [] - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { @@ -8656,7 +8656,7 @@ export class Brainy implements BrainyInterface { if (params.fields !== undefined && params.fields.length > 0 && result.length > 0) { const named = [...new Set(params.fields)] result = result.map((r) => { - const projected = this.projectEntity( + const projected = this.#projectEntity( r.id, named, undefined, @@ -17038,7 +17038,7 @@ export class Brainy implements BrainyInterface { ordered = valued.map((v) => v.id) } const pageIds = ordered.slice(offset, offset + limit) - const entitiesMap = await this.hydratePage(pageIds, params.fields, this.guardFieldsFor(params)) + const entitiesMap = await this.#hydratePage(pageIds, params.fields, this.#guardFieldsFor(params)) const results: Result[] = [] for (const id of pageIds) { const entity = entitiesMap.get(id) diff --git a/tests/integration/find-fields-projection.test.ts b/tests/integration/find-fields-projection.test.ts index 3718dc4f..df94e942 100644 --- a/tests/integration/find-fields-projection.test.ts +++ b/tests/integration/find-fields-projection.test.ts @@ -172,21 +172,32 @@ describe('find/get({ fields }) — projection', () => { expect(reads).toBe(0) }) - it('reads records only for the rows that owe an un-served field', async () => { - // `body` is not a scalar the index serves, so the record must be opened — - // but the projection still returns only the named fields. + it('reads records only for the fields the column cannot serve', async () => { + // `system.data` is NOT a column the store holds (verified against + // getIndexedFields), so the record must be opened for it — while `title`, + // which the column does hold, still comes from the index. const { out, reads } = await countingReads(() => - brain.find({ where: { kind: 'post' }, fields: ['title', 'body'], limit: 4 }) + brain.find({ where: { kind: 'post' }, fields: ['title', 'system.data'], limit: 4 }) ) expect(out).toHaveLength(4) expect(reads).toBe(4) for (const r of out) { const meta = (r.entity.metadata ?? {}) as Record - expect(meta.body).toBe(BODY) - expect(Object.keys(meta).sort()).toEqual(['body', 'title']) + expect(Object.keys(meta)).toEqual(['title']) + expect(typeof (r.entity as any).data).toBe('string') } }) + it('a large field the column DOES hold costs no record read', async () => { + // Worth pinning because it is the venue case: the body is column-served on + // this engine, so a list that projects around it pays nothing for it, and + // a list that projects it still pays no record read. + const { reads } = await countingReads(() => + brain.find({ where: { kind: 'post' }, fields: ['body'], limit: 4 }) + ) + expect(reads).toBe(0) + }) + it('get({ fields }) projects a single row through the same seam', async () => { const full = await brain.get(ids[0]) const projected = await brain.get(ids[0], { fields: ['title', 'slug'] }) @@ -205,20 +216,32 @@ describe('find/get({ fields }) — projection', () => { expect(reads).toBe(0) }) - it('the provider door serves only what it can serve EXACTLY', async () => { - // The bucketed timestamps are indexed at minute precision for range - // queries. The door must omit them rather than hand back a bucket that - // differs from the record — omission costs a read, a wrong value is a wrong - // answer nobody can see. + it('the door serves EXACT values — the column, never the bucketed index', async () => { + // The sparse index buckets `system.createdAt` to the minute for range + // queries; the column store keeps raw ms. Serving a projection from the + // former would hand back a value that differs from the record's, so the + // door reads the column — and this pin is what proves which one it read. const index = (brain as any).metadataIndex - const served = await index.getScalarsForIds(ids.slice(0, 3), [ - 'title', - 'system.createdAt' - ]) - expect(served.size).toBeGreaterThan(0) + const sample = ids.slice(0, 3) + const served = await index.getScalarsForIds(sample, ['title', 'system.createdAt']) + expect(served.size).toBe(sample.length) + for (const id of sample) { + const row = served.get(id)! + const record = await brain.get(id) + expect(row.title).toEqual((record!.metadata as any).title) + // Exact to the millisecond — a bucketed value would be rounded down to + // the minute and this would fail. + expect(row['system.createdAt']).toEqual((record as any).createdAt) + } + }) + + it('a field the column store does not hold is OMITTED, not approximated', async () => { + const index = (brain as any).metadataIndex + const served = await index.getScalarsForIds(ids.slice(0, 2), ['title', 'system.data']) for (const [, row] of served) { expect('title' in row).toBe(true) - expect('system.createdAt' in row).toBe(false) + // Omission is what makes the caller read the record for it. + expect('system.data' in row).toBe(false) } }) }) From 69bda5b7cb6e7c730c15be4dcc5b614c5d24f626 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:50:19 -0700 Subject: [PATCH 125/150] =?UTF-8?q?test(find):=20a=20vector-leg=20find=20i?= =?UTF-8?q?s=20projected=20too=20=E2=80=94=20the=20answer=20is=20uniform?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seam hydrates the metadata and graph page paths; a vector or text leg builds its own entities and is trimmed after the integrity guard instead. That is a COST difference, and this pin exists so it can never quietly become an ANSWER difference. --- tests/integration/find-fields-projection.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/find-fields-projection.test.ts b/tests/integration/find-fields-projection.test.ts index df94e942..5d339f08 100644 --- a/tests/integration/find-fields-projection.test.ts +++ b/tests/integration/find-fields-projection.test.ts @@ -198,6 +198,20 @@ describe('find/get({ fields }) — projection', () => { expect(reads).toBe(0) }) + it('projects a vector-leg find too — the ANSWER is uniform, only the cost is not', async () => { + // The seam hydrates the metadata and graph page paths. A vector or text leg + // builds its own entities, so those rows are trimmed after the integrity + // guard instead. That difference is a COST difference, and this pin exists + // so it can never quietly become an ANSWER difference. + const rows = await brain.find({ query: 'post', fields: ['title'], limit: 3 }) + for (const r of rows) { + const meta = (r.entity.metadata ?? {}) as Record + expect(Object.keys(meta)).toEqual(['title']) + expect(meta.body).toBeUndefined() + expect(r.entity.id).toBe(r.id) + } + }) + it('get({ fields }) projects a single row through the same seam', async () => { const full = await brain.get(ids[0]) const projected = await brain.get(ids[0], { fields: ['title', 'slug'] }) From 5e720d17ae2f1b6a308e83a4bb37e83a9ba0ad0e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:30:20 -0700 Subject: [PATCH 126/150] fix(find): orderBy is the order on every path, not only the metadata-only one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `find({ where, orderBy })` answered in field order; `find({ query, where, orderBy })` and `find({ vector, where, orderBy })` answered in SCORE order. The vector/filter block ranks the fused candidates by score, cuts the page and returns early — and the tail's orderBy sort sits below that early return, so on those paths it never ran. Nothing threw and nothing warned: the ordering request was dropped in silence, and the two paths disagreed about what "ordered by rank" means. Where `connected` or `fusion` kept the tail alive the defect changed shape rather than disappearing. The block had already CUT the page by score, so the tail ordered the rows relevance had chosen — a correctly sorted page of the wrong rows. The early cut fires only once the candidate set reaches offset+limit rows, which is why it read green for so long: below that threshold the block falls through and the tail's sort does apply. An ordering that is correct until there is enough data to matter. THE LAW: an explicit orderBy displaces score as the ordering key on every path. The candidate set the path produced is ordered IN FULL and the page is cut from that ordering — "page last", the graph-first law applied to ordering rather than to filtering. Score-ranked early paging stays exactly as it was for the default case, where score IS the requested order. The pin is differential against the metadata-only path, the one path that always honoured orderBy. It is sized so the hybrid legs (each bounded at limit*2) provably cover the filter universe, and that covering is asserted from the leg's own output rather than assumed — orderBy orders the candidate set, it does not enlarge it, and the pin claims nothing about recall. --- src/brainy.ts | 18 +- .../find-orderby-every-path.test.ts | 244 ++++++++++++++++++ 2 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 tests/integration/find-orderby-every-path.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index edcba3d4..a97bdde2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -8486,7 +8486,23 @@ export class Brainy implements BrainyInterface { // Rank by score (top offset+limit), then drop the offset — identical ordering // to a full `sort((a, b) => b.score - a.score)` + slice, but the native // `sort:topK` provider can compute only the page instead of the full sort. - if (results.length >= offset + limit) { + // + // ONLY when score IS the requested order. An explicit `orderBy` names a + // different ordering key, and this block cannot serve it: it ranks by + // score and CUTS the page, so the tail's `orderBy` sort below either + // never runs at all (the early return, when there is no `connected` / + // `fusion` work left) or runs over a page that score already chose — + // ordering eight rows relevance picked instead of the eight the field + // ordering asks for. Both readings were silent: `find({ query, where, + // orderBy })` answered in score order while `find({ where, orderBy })` + // answered in field order, and nothing said the request had been dropped. + // + // With `orderBy` present the candidate set falls through UNCUT to the + // tail, which orders it in full and pages that ordering — "page last", + // the graph-first law applied to ordering rather than to filtering. The + // set is bounded by the legs (the text matches inside the universe plus + // the beam walk's `limit * 2`), not by the store. + if (!params.orderBy && results.length >= offset + limit) { const k = offset + limit const order = rankIndicesByScore(results.map(r => r.score), k, true) results = reorderByIndices(results, order).slice(offset, k) diff --git a/tests/integration/find-orderby-every-path.test.ts b/tests/integration/find-orderby-every-path.test.ts new file mode 100644 index 00000000..7637a79b --- /dev/null +++ b/tests/integration/find-orderby-every-path.test.ts @@ -0,0 +1,244 @@ +/** + * @module tests/integration/find-orderby-every-path + * @description `orderBy` IS THE ORDER — on every find() path, not just the + * metadata-only one. + * + * THE DEFECT. `find({ where, orderBy })` (metadata only) answered in field + * order. `find({ query, where, orderBy })` and `find({ vector, where, orderBy })` + * answered in SCORE order, silently: the vector/filter block ranked the fused + * candidates by score, cut the page, and returned early — the tail's `orderBy` + * sort sat below that early return and never ran. Nothing threw, nothing warned, + * and the two paths disagreed about what "ordered by rank" means. A caller + * paging `orderBy: 'rank', order: 'desc'` over a hybrid find got relevance + * order wearing an ordering request's clothes. + * + * Where `connected` or `fusion` kept the tail alive the defect changed shape + * rather than disappearing: the block had already CUT the page by score, so the + * tail ordered the rows relevance had chosen instead of the rows the ordering + * asks for — a correctly sorted page of the wrong rows. + * + * The early cut fires only once the candidate set reaches `offset + limit` + * rows, which is why small fixtures never saw it: below that threshold the + * block falls through and the tail's sort does apply. That is the whole shape + * of the bug — an ordering that is correct until there is enough data to matter. + * + * THE LAW. An explicit `orderBy` displaces score as the ordering key on every + * path. The candidate set the path produced is ordered IN FULL and the page is + * cut from that ordering — the graph-first law's "page last", applied to + * ordering rather than to filtering. Score-ranked early paging is for the + * default (no `orderBy`) case only, where score IS the requested order. + * + * THE PIN. Differential, against the metadata-only path — the one path that + * always honoured `orderBy`. + * + * WHAT THE DIFFERENTIAL CAN AND CANNOT CLAIM. `orderBy` orders the candidate + * set; it does not enlarge it. The hybrid legs are bounded by construction (the + * text leg and the beam walk each take `limit * 2`), so a differential against + * the metadata-only path — whose universe is every matching row — is only + * meaningful where those bounds provably cover the universe. The fixture is + * sized so they do (12 rows, `limit` 6 → a `limit * 2` = 12-row text leg), and + * the covering is ASSERTED from the leg's own output rather than assumed. This + * pin is about ordering, and it says nothing about recall. + */ +import { describe, it, expect, beforeAll } from 'vitest' +import { Brainy } from '../../src/brainy' +import { NounType, VerbType } from '../../src/types/graphTypes' +import { resolveEntityId } from '../../src/utils/idNormalization' + +/** Embedding width of the default model — the row vectors must match it. */ +const DIM = 384 + +/** A deterministic, per-row-distinct unit vector (no embedder in the fixture). */ +function seededVector(seed: number): number[] { + const v = new Array(DIM) + for (let i = 0; i < DIM; i++) { + v[i] = Math.sin((i + 1) * 0.11 + seed * 0.37) * 0.5 + Math.cos((i + 1) * 0.05 + seed * 0.13) * 0.3 + } + const magnitude = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0)) + return v.map((x) => x / magnitude) +} + +/** + * Ranks, shuffled — so no scoring order can reproduce them by luck, and the + * ordering the pins assert is visibly not the insertion order either. + */ +const RANKS = [7, 3, 11, 1, 9, 5, 12, 2, 10, 4, 8, 6] +const ROWS = RANKS.length +/** The page size every pin uses: `limit * 2` covers the whole universe. */ +const LIMIT = 6 +/** The neighbour subset — the graph-first universe — and its own page size. */ +const NEIGHBOURS = 8 +const GRAPH_LIMIT = 4 + +describe('find(): orderBy is the order on every path', () => { + let brain: Brainy + const QUERY = 'orbital telemetry' + const anchor = 'ordering-anchor' + const neighbourIds: string[] = [] + + beforeAll(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + + let seed = 1 + await brain.add({ + id: anchor, + data: 'ground station anchor record', + type: NounType.Thing, + metadata: { lane: 'anchor', rank: 0 }, + vector: seededVector(seed++) + }) + + for (let i = 0; i < ROWS; i++) { + const id = `row-${i}` + await brain.add({ + id, + // EVERY row carries both query words, so the text leg reaches all of + // them and the hybrid candidate set covers the whole universe. + data: `orbital telemetry packet ${i} recorded downlink`, + type: NounType.Document, + metadata: { lane: 'alpha', rank: RANKS[i] }, + vector: seededVector(seed++) + }) + if (i < NEIGHBOURS) { + await brain.relate({ from: anchor, to: id, type: VerbType.RelatedTo }) + neighbourIds.push(resolveEntityId(id)) + } + } + }) + + it('the fixture: the hybrid candidate set covers the whole filter universe', async () => { + const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) + expect(universe).toHaveLength(ROWS) + + // The text leg is bounded at `limit * 2`; the fixture is sized so that + // bound reaches every row in the universe. This is the precondition the + // differential below rests on — asserted from the leg itself. + const textScored = await (brain as any).executeTextSearchScored(QUERY, LIMIT * 2, universe) + expect(textScored).toHaveLength(ROWS) + + // And the candidate set is large enough to trigger the score-ranked early + // cut this pin exists to keep out of an ordered query's way. + expect(ROWS).toBeGreaterThanOrEqual(LIMIT) + }) + + it('metadata-only + orderBy: the reference ordering', async () => { + const rows = await brain.find({ + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc', + limit: LIMIT + } as any) + expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) + }) + + it('hybrid (query + where) + orderBy: the same page as the metadata-only path', async () => { + const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'desc' as const, limit: LIMIT } + const expected = await brain.find(params as any) + const actual = await brain.find({ ...params, query: QUERY } as any) + + expect(actual).toHaveLength(expected.length) + expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) + expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) + }) + + it('hybrid + orderBy asc: the ordering key is honoured in both directions', async () => { + const params = { where: { lane: 'alpha' }, orderBy: 'rank', order: 'asc' as const, limit: LIMIT } + const expected = await brain.find(params as any) + const actual = await brain.find({ ...params, query: QUERY } as any) + + expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) + expect(actual.map((r: any) => r.metadata.rank)).toEqual([1, 2, 3, 4, 5, 6]) + }) + + it('hybrid + orderBy + offset: page two is page two of the ORDERING', async () => { + const params = { + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc' as const, + limit: LIMIT, + offset: LIMIT + } + const expected = await brain.find(params as any) + const actual = await brain.find({ ...params, query: QUERY } as any) + + expect(actual).toHaveLength(LIMIT) + expect(actual.map((r: any) => r.id)).toEqual(expected.map((r: any) => r.id)) + expect(actual.map((r: any) => r.metadata.rank)).toEqual([6, 5, 4, 3, 2, 1]) + }) + + it('hybrid + orderBy: paging walks the ordering monotonically, no row twice', async () => { + const seen: number[] = [] + for (let offset = 0; offset < ROWS; offset += LIMIT) { + const page = await brain.find({ + query: QUERY, + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc', + limit: LIMIT, + offset + } as any) + seen.push(...page.map((r: any) => r.metadata.rank)) + } + expect(seen).toHaveLength(ROWS) + expect(new Set(seen).size).toBe(ROWS) + // Strictly descending across every page boundary. + for (let i = 1; i < seen.length; i++) expect(seen[i]).toBeLessThan(seen[i - 1]) + }) + + it('vector + where + orderBy: field order, not distance order', async () => { + // The beam walk takes `limit * 2` = the whole universe here, so the page is + // the true top of the ordering — which distance order cannot produce. + const rows = await brain.find({ + vector: seededVector(1000), + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc', + limit: LIMIT + } as any) + expect(rows.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) + }) + + it('graph-first (query + connected + where) + orderBy: the neighbour set, ordered', async () => { + const actual = await brain.find({ + query: QUERY, + connected: { from: anchor, direction: 'out' as const }, + where: { lane: 'alpha' }, + orderBy: 'rank', + order: 'desc', + limit: GRAPH_LIMIT + } as any) + + expect(actual).toHaveLength(GRAPH_LIMIT) + const neighbours = new Set(neighbourIds) + for (const r of actual) expect(neighbours.has(r.id)).toBe(true) + + // The ordering covers the whole neighbour set, so the page holds the + // highest ranks AMONG THE NEIGHBOURS — not the ones the score ranking + // happened to surface first and the tail then sorted among themselves. + const expectedRanks = RANKS.slice(0, NEIGHBOURS) + .sort((a, b) => b - a) + .slice(0, GRAPH_LIMIT) + expect(expectedRanks).toEqual([12, 11, 9, 7]) + expect(actual.map((r: any) => r.metadata.rank)).toEqual(expectedRanks) + }) + + it('fusion + orderBy: the ordering survives the fusion rescore', async () => { + const actual = await brain.find({ + query: QUERY, + where: { lane: 'alpha' }, + fusion: 'weighted', + orderBy: 'rank', + order: 'desc', + limit: LIMIT + } as any) + expect(actual.map((r: any) => r.metadata.rank)).toEqual([12, 11, 10, 9, 8, 7]) + }) + + it('no orderBy: score order still stands (the default is untouched)', async () => { + const rows = await brain.find({ query: QUERY, where: { lane: 'alpha' }, limit: LIMIT } as any) + expect(rows).toHaveLength(LIMIT) + const scores = rows.map((r: any) => r.score) + for (let i = 1; i < scores.length; i++) expect(scores[i]).toBeLessThanOrEqual(scores[i - 1]) + }) +}) From a7eb7f5222b2e8a72cb170cf2bb11724d951b42e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:37:08 -0700 Subject: [PATCH 127/150] =?UTF-8?q?fix(metadata):=20the=20legacy=20sparse?= =?UTF-8?q?=20range=20path=20orders=20values,=20or=20refuses=20=E2=80=94?= =?UTF-8?q?=20never=20ranks=20by=20hash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `getIdsForRange` routes two ways. The column store compares RAW values and is correct. The legacy sparse chunk index — the pre-7.20.0 fallback, still read for workspaces that have not been rebuilt — compared normalizeValue() output, and normalizeValue carries an escape hatch that destroys order on purpose: a string over 100 characters becomes a short hash so it can serve as a filesystem-safe key. Ordering hashes ranks rows by digest. Two shapes, both silent: (a) A LONG BOUND against ordinary values. `{ gte: }` collapsed the BOUND to `__HASH_…`, whose leading underscores sort below every letter — so a bound that should have excluded everything matched the entire field instead. Measured on the fixture here: 3 of 3 rows returned where 0 is correct. This shape reaches a caller who never stored a long value at all. (b) LONG VALUES in the index. The field was persisted hashed, so its order is not recoverable from this index. The old code compared the digests anyway and returned a subset chosen by hash — 1 of 3 rows, the wrong one. Bounds are now normalized WITHOUT the hash escape hatch, so a long bound stays comparable and (a) is simply fixed. Where the persisted KEY is a hash the order does not exist to be computed, and the query throws a typed BrainyError('INVALID_QUERY') naming the field and the cure. The refusal is checked before chunk SELECTION as well as during the scan: selection orders the bounds against each chunk's zone map, and its failure mode is an empty answer — the quietest wrong answer of all. Equality on a hashed field is untouched; only ordering is refused. KNOWN, NAMED DIVERGENCE, recorded in the doc comment rather than papered over: the persisted keys are also lower-cased and trimmed, so this path's string ranges are case-INSENSITIVE where the column store's are not. The raw values are not in the index to compare — that is a property of the bytes a pre-7.20.0 engine wrote, and it ends when the column store adopts the field. The pin builds a genuine legacy index through the same ChunkManager / SparseIndex doors that engine wrote through, into a field the column store does not serve. The chunk write path was removed in 11be039, so that is the only way to build the shape this read path exists for. --- src/utils/metadataIndex.ts | 118 +++++++- ...tadataIndex-sparse-range-collation.test.ts | 258 ++++++++++++++++++ 2 files changed, 366 insertions(+), 10 deletions(-) create mode 100644 tests/unit/utils/metadataIndex-sparse-range-collation.test.ts diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index d07fa8d7..83d37379 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -961,9 +961,41 @@ export class MetadataIndexManager implements MetadataIndexProvider { } /** - * Get IDs for a range using chunked sparse index with zone maps and roaring bitmaps - * Now fully lazy-loaded via UnifiedCache (no local sparseIndices Map) - * Normalize min/max for timestamp bucketing before comparison + * Get IDs for a range using the legacy chunked sparse index (zone maps + + * roaring bitmaps). Lazy-loaded via UnifiedCache. + * + * ORDER IS NOT A KEY. This path compares NORMALIZED values, and + * {@link normalizeValue} carries an escape hatch that is order-destroying by + * design: a string over 100 characters is replaced by {@link hashValue}'s + * digest so it can be used as a filesystem-safe key. Feeding that digest to + * an ORDERING comparison — which is what a `gte` / `lt` / `between` does — + * ranks rows by hash. The result is not empty and not an error: it is a + * confidently ordered wrong answer, and it disagrees with the column-store + * path (`getIdsForRange` above), which compares raw values and is correct. + * + * Two changes hold the line here: + * + * 1. THE BOUNDS ARE NEVER HASHED. They are normalized with `allowHash = + * false`, so a long bound stays comparable instead of collapsing to a + * digest. This alone fixes the common shape — a long bound queried + * against ordinary short values, where the digest sorts below every + * letter and `gte` therefore matched the entire store. + * + * 2. A HASHED KEY IS REFUSED, NEVER GUESSED. The persisted keys are whatever + * the pre-7.20.0 writer normalized them to, so a field whose values ran + * long is stored hashed and its order is simply not recoverable from this + * index. Rather than compare digests, the query throws a typed + * `BrainyError('INVALID_QUERY')` naming the field, the bound and the cure. + * Loud beats wrong. + * + * KNOWN, NAMED DIVERGENCE. The persisted keys are also lower-cased and + * trimmed by `normalizeValue`, so this path's string ranges are + * CASE-INSENSITIVE where the column store's are not. That is a property of + * the bytes a pre-7.20.0 engine wrote, not of the comparison: the raw values + * are not in the index to compare. The bounds are normalized into the same + * case-folded space so the comparison is at least self-consistent, and the + * divergence disappears with the field itself once the column store adopts + * it. See the module note on `getIdsFromChunks` for the path's lifetime. */ private async getIdsFromChunksForRange( field: string, @@ -979,9 +1011,27 @@ export class MetadataIndexManager implements MetadataIndexProvider { } // Normalize min/max for consistent comparison with indexed values - // (indexed values are bucketed for timestamps, so we must bucket the query bounds too) - const normalizedMin = min !== undefined ? this.normalizeValue(min, field) : undefined - const normalizedMax = max !== undefined ? this.normalizeValue(max, field) : undefined + // (indexed values are bucketed for timestamps, so we must bucket the query + // bounds too) — but NEVER through the hash escape hatch, which would make + // the bound incomparable. See the doc comment above. + const normalizedMin = min !== undefined ? this.normalizeValue(min, field, false) : undefined + const normalizedMax = max !== undefined ? this.normalizeValue(max, field, false) : undefined + + // REFUSE BEFORE SELECTING. Chunk selection itself orders values: it tests + // the bounds against each chunk's zone-map min/max. If those are hashes the + // selection is already meaningless — and its failure mode is an EMPTY + // answer (no chunk appears to overlap), which is the quietest wrong answer + // of all. So the key space is checked here, before a single chunk is + // chosen, and again per key below for a chunk whose zone map happens to + // read clean. + for (const chunkId of sparseIndex.getAllChunkIds()) { + const zoneMap = sparseIndex.getChunk(chunkId)?.zoneMap + for (const bound of [zoneMap?.min, zoneMap?.max]) { + if (typeof bound === 'string' && MetadataIndexManager.isHashedValue(bound)) { + throw MetadataIndexManager.rangeOverHashedIndex(field) + } + } + } // Find candidate chunks using zone maps const candidateChunkIds = sparseIndex.findChunksForRange(normalizedMin, normalizedMax) @@ -996,6 +1046,13 @@ export class MetadataIndexManager implements MetadataIndexProvider { const chunk = await this.chunkManager.loadChunk(field, chunkId) if (chunk) { for (const [value, bitmap] of chunk.entries) { + // A hashed key carries no order. Refuse the range rather than rank by + // digest — the whole answer is unsound, so failing on the first one + // is the honest outcome. + if (MetadataIndexManager.isHashedValue(value)) { + throw MetadataIndexManager.rangeOverHashedIndex(field) + } + // Check if value is in range using numeric-aware comparison // (normalizeValue converts numbers to strings, so we must compare numerically) let inRange = true @@ -1024,6 +1081,25 @@ export class MetadataIndexManager implements MetadataIndexProvider { return this.idMapper.intsIterableToUuids(allIntIds) } + /** + * The refusal a range query gets when the legacy sparse index holds hashed + * keys for the field. Names the field and the cure; never a wrong answer. + */ + private static rangeOverHashedIndex(field: string): BrainyError { + return new BrainyError( + `Range query on field "${field}" cannot be served by the legacy sparse index: ` + + `its values were persisted as hashes (values over 100 characters are stored ` + + `hashed to stay within filesystem name limits), and a hash carries no order — ` + + `comparing them would return a confidently ordered wrong answer. ` + + `Equality (\`where: { ${field}: value }\`) still works on this index. ` + + `To range over this field, let the column store adopt it: run ` + + `brain.repairIndex({ rebuild: ['metadata'] }), which rebuilds the field into ` + + `the column store, where ranges compare raw values.`, + 'INVALID_QUERY', + false + ) + } + /** * Get roaring bitmap for a field-value pair without converting to UUIDs * This is used for fast multi-field intersection queries using hardware-accelerated bitmap AND @@ -1191,8 +1267,17 @@ export class MetadataIndexManager implements MetadataIndexProvider { * value-based detection (DuckDB-inspired). Analyzes actual data values, not names. * * NO FALLBACKS - Pure value-based detection only. + * + * @param value - The value to normalize. + * @param field - Optional field name (drives the per-field statistics strategy). + * @param allowHash - Whether the >100-character escape hatch may fire. TRUE + * everywhere a normalized value is used as a KEY (equality postings, chunk + * entries, filenames) — that is what the hash exists for. FALSE on the + * ORDER-comparing path: a hash is deliberately order-destroying, so a + * bound that hashes can only be compared as nonsense. See + * {@link isHashedValue} and `getIdsFromChunksForRange`. */ - private normalizeValue(value: any, field?: string): string { + private normalizeValue(value: any, field?: string, allowHash: boolean = true): string { if (value === null || value === undefined) return '__NULL__' if (typeof value === 'boolean') return value ? '__TRUE__' : '__FALSE__' @@ -1250,21 +1335,34 @@ export class MetadataIndexManager implements MetadataIndexProvider { // Default normalization if (typeof value === 'number') return value.toString() if (Array.isArray(value)) { - const joined = value.map(v => this.normalizeValue(v, field)).join(',') + const joined = value.map(v => this.normalizeValue(v, field, allowHash)).join(',') // Hash very long array values to avoid filesystem limits - if (joined.length > 100) { + if (allowHash && joined.length > 100) { return this.hashValue(joined) } return joined } const stringValue = String(value).toLowerCase().trim() // Hash very long string values to avoid filesystem limits - if (stringValue.length > 100) { + if (allowHash && stringValue.length > 100) { return this.hashValue(stringValue) } return stringValue } + /** + * Is this normalized value a HASH rather than the value itself? + * + * {@link hashValue} is an escape hatch for filesystem name limits, and it is + * deliberately order-destroying: two values whose hashes compare one way + * routinely compare the other way themselves. Anything that ORDERS normalized + * values has to know when it is holding one, because comparing hashes yields + * a confident, wrong answer rather than an error. + */ + private static isHashedValue(normalized: string): boolean { + return normalized.startsWith('__HASH_') + } + /** * Create a short hash for long values to avoid filesystem filename limits */ diff --git a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts new file mode 100644 index 00000000..d6d00568 --- /dev/null +++ b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts @@ -0,0 +1,258 @@ +/** + * @module tests/unit/utils/metadataIndex-sparse-range-collation + * @description RANGE QUERIES ON THE LEGACY SPARSE INDEX — order, or a refusal. + * Never a confidently ordered wrong answer. + * + * THE TWO RANGE PATHS. `getIdsForRange` routes a `gte` / `lt` / `between` two + * ways. The column store compares RAW values and is correct. The legacy sparse + * chunk index — the pre-7.20.0 fallback, still read for workspaces that have + * not been rebuilt — compared `normalizeValue()` output, and `normalizeValue` + * carries an escape hatch that destroys order on purpose: a string over 100 + * characters is replaced by a short hash so it can serve as a filesystem-safe + * key. Ordering hashes ranks rows by digest. + * + * THE DEFECT, IN TWO SHAPES. + * + * (a) A LONG BOUND against ordinary values. `where: { title: { gte: } }` collapsed the BOUND to `__HASH_…`, whose + * leading underscores sort below every letter — so a bound that should + * have excluded everything matched the entire field instead. This is the + * shape that reaches a caller who never stored a long value at all. + * + * (b) LONG VALUES in the index. A field whose values ran long was persisted + * hashed, so its order is not recoverable from this index at all. The old + * code compared the digests anyway and returned a subset chosen by hash. + * + * THE LAW. Bounds are normalized WITHOUT the hash escape hatch, so a long + * bound stays comparable — (a) is simply fixed. Where the persisted KEY is a + * hash, the order does not exist to be computed, and the query throws a typed + * `BrainyError('INVALID_QUERY')` naming the field and the cure — (b) is + * refused by name. Loud beats wrong. + * + * THE FIXTURE is a genuine legacy index: it is written through the same + * `ChunkManager` / `SparseIndex` doors a pre-7.20.0 engine wrote through, with + * keys normalized exactly as that engine normalized them, into a field the + * column store does not serve. The chunk WRITE path was removed in 11be039, so + * this is the only way the shape the read path exists for can be built. + * + * NOT CLAIMED HERE. The persisted keys are also lower-cased and trimmed by + * `normalizeValue`, so this path's string ranges are case-INSENSITIVE where + * the column store's are not. The raw values are not in the index to compare — + * that divergence is a property of the bytes on disk and it ends when the + * column store adopts the field. It is named in `getIdsFromChunksForRange`'s + * doc comment rather than papered over. + */ +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' +import { SparseIndex, ChunkManager } from '../../../src/utils/metadataIndexChunking' +import { BrainyError } from '../../../src/errors/brainyError' + +/** The field the legacy index covers — deliberately never given to a row, so + * the column store never learns it and the sparse fallback is the only path. */ +const FIELD = 'legacyTitle' + +/** + * Write a legacy sparse index for `field` exactly as a pre-7.20.0 engine did: + * one chunk, keys normalized through the index's own `normalizeValue`, ids as + * roaring bitmaps, a zone map and a bloom filter over the chunk. + * + * @param brain - The live brain whose metadata index gains the legacy field. + * @param field - Field name to index. + * @param valueToIds - Raw value → the entity ids that carried it. + */ +async function writeLegacySparseIndex( + brain: any, + field: string, + valueToIds: Array<[string, string[]]> +): Promise { + const index = brain.metadataIndex + const chunkManager: ChunkManager = index.chunkManager + const sparseIndex = new SparseIndex(field) + + // The keys a pre-7.20.0 writer persisted: normalizeValue output, hash escape + // hatch and all. This is what makes the fixture the real shape. + const chunk = await chunkManager.createChunk(field) + for (const [value, ids] of valueToIds) { + const key = index.normalizeValue(value, field) + for (const id of ids) await chunkManager.addToChunk(chunk, key, id) + } + await chunkManager.saveChunk(chunk) + + sparseIndex.registerChunk( + { + chunkId: chunk.chunkId, + field, + valueCount: chunk.entries.size, + idCount: Array.from(chunk.entries.values()).reduce((s: number, b: any) => s + b.size, 0), + zoneMap: (chunkManager as any).calculateZoneMap(chunk), + lastUpdated: Date.now(), + splitThreshold: 80, + mergeThreshold: 20 + }, + chunkManager.createBloomFilter(chunk) + ) + + await index.saveSparseIndex(field, sparseIndex) +} + +/** A deterministic string of `n` characters starting with `lead`. */ +function longString(lead: string, n: number): string { + return lead + 'x'.repeat(n - lead.length) +} + +describe('legacy sparse index: range queries order values, or refuse', () => { + let brain: Brainy + let index: any + let ids: string[] + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + index = (brain as any).metadataIndex + + // Rows exist (so the id mapper can resolve them) but carry NO `legacyTitle` + // — the column store must not serve the field the pins query. + ids = [] + for (let i = 0; i < 3; i++) { + const id = `row-${i}` + await brain.add({ id, data: `row ${i}`, type: NounType.Thing, metadata: { lane: 'a' }, vector: [] }) + ids.push(id) + } + expect(index.columnStore.hasField(FIELD)).toBe(false) + }) + + describe('(a) a long BOUND against ordinary short values', () => { + // 'apple' < 'mango' < 'zebra', and every bound below is compared against + // these three raw keys. + beforeEach(async () => { + await writeLegacySparseIndex(brain, FIELD, [ + ['apple', [ids[0]]], + ['mango', [ids[1]]], + ['zebra', [ids[2]]] + ]) + }) + + it('the fixture: the values are stored raw, the long bound is what hashes', () => { + expect(index.normalizeValue('apple', FIELD)).toBe('apple') + // The bound is what the old code collapsed — and a digest sorts below + // every letter, which is exactly why `gte` matched everything. + const bound = longString('zzz', 120) + expect(index.normalizeValue(bound, FIELD)).toMatch(/^__HASH_/) + expect(index.normalizeValue(bound, FIELD) < 'apple').toBe(true) + }) + + it('gte a bound above every value matches NOTHING (it used to match all)', async () => { + const bound = longString('zzz', 120) + const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) + expect(matched).toEqual([]) + }) + + it('lte a bound above every value matches EVERY value', async () => { + const bound = longString('zzz', 120) + const matched = await index.getIdsForRange(FIELD, undefined, bound, true, true) + expect(matched).toHaveLength(3) + }) + + it('gte a long bound below every value matches every value', async () => { + const bound = longString('aaa', 120) + const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) + expect(matched).toHaveLength(3) + }) + + it('a long bound orders BETWEEN the values, not below all of them', async () => { + // 'mmm…' sits between 'mango' and 'zebra'. + const bound = longString('mmm', 120) + const matched = await index.getIdsForRange(FIELD, bound, undefined, true, true) + expect(matched).toHaveLength(1) + }) + + it('short bounds are unchanged — the ordinary case still orders correctly', async () => { + expect(await index.getIdsForRange(FIELD, 'b', undefined, true, true)).toHaveLength(2) + expect(await index.getIdsForRange(FIELD, undefined, 'n', true, true)).toHaveLength(2) + expect(await index.getIdsForRange(FIELD, 'b', 'n', true, true)).toHaveLength(1) + // Strict bounds stay strict. + expect(await index.getIdsForRange(FIELD, 'mango', undefined, false, true)).toHaveLength(1) + expect(await index.getIdsForRange(FIELD, 'mango', undefined, true, true)).toHaveLength(2) + }) + }) + + describe('(b) long VALUES — the index holds hashes, so the range is refused', () => { + beforeEach(async () => { + await writeLegacySparseIndex(brain, FIELD, [ + [longString('alpha', 140), [ids[0]]], + [longString('mike', 140), [ids[1]]], + [longString('zulu', 140), [ids[2]]] + ]) + }) + + it('the fixture: the persisted keys really are hashes', async () => { + const chunk = await index.chunkManager.loadChunk(FIELD, 0) + const keys = Array.from(chunk.entries.keys()) as string[] + expect(keys).toHaveLength(3) + for (const k of keys) expect(k).toMatch(/^__HASH_/) + // And their digest order is NOT their value order — the wrong answer the + // old code returned was wrong, not merely arbitrary. + const digestOrder = [...keys].sort() + const valueOrder = [ + index.normalizeValue(longString('alpha', 140), FIELD), + index.normalizeValue(longString('mike', 140), FIELD), + index.normalizeValue(longString('zulu', 140), FIELD) + ] + expect(digestOrder).not.toEqual(valueOrder) + }) + + it('a range over the hashed field throws a typed refusal naming the field', async () => { + await expect( + index.getIdsForRange(FIELD, longString('mike', 140), undefined, true, true) + ).rejects.toThrow(BrainyError) + + const err = await index + .getIdsForRange(FIELD, longString('mike', 140), undefined, true, true) + .catch((e: any) => e) + expect(err).toBeInstanceOf(BrainyError) + expect(err.type).toBe('INVALID_QUERY') + expect(err.message).toContain(FIELD) + expect(err.message).toContain('hash') + // The cure is named, not left to the caller to guess. + expect(err.message).toContain('repairIndex') + }) + + it('every range shape refuses — gte, lte and between alike', async () => { + const lo = longString('alpha', 140) + const hi = longString('zulu', 140) + for (const [min, max] of [ + [lo, undefined], + [undefined, hi], + [lo, hi] + ] as Array<[any, any]>) { + const err = await index.getIdsForRange(FIELD, min, max, true, true).catch((e: any) => e) + expect(err).toBeInstanceOf(BrainyError) + expect(err.type).toBe('INVALID_QUERY') + } + }) + + it('EQUALITY still works on the hashed field — only ordering is refused', async () => { + const matched = await index.getIds(FIELD, longString('mike', 140)) + expect(matched).toHaveLength(1) + }) + }) + + describe('numeric ranges on the legacy path are untouched', () => { + beforeEach(async () => { + await writeLegacySparseIndex(brain, FIELD, [ + ['5', [ids[0]]], + ['50', [ids[1]]], + ['500', [ids[2]]] + ]) + }) + + it('numbers still compare numerically, not lexicographically', async () => { + // The whole point of compareNormalizedValues: "50" < "500" numerically + // even though "500" < "50" would hold as strings by prefix. + expect(await index.getIdsForRange(FIELD, 10, undefined, true, true)).toHaveLength(2) + expect(await index.getIdsForRange(FIELD, undefined, 100, true, true)).toHaveLength(2) + expect(await index.getIdsForRange(FIELD, 10, 100, true, true)).toHaveLength(1) + }) + }) +}) From 0d5ab6077da73a27946b54efaac0e5baae2e95c6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:43:14 -0700 Subject: [PATCH 128/150] fix(metadata): the indexable-array bound is a named law with a refusal, not a silent skip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An array-valued metadata field indexes one posting per element, so the index has always carried a ceiling. It was 10, and it was applied by a bare `continue` deep inside field extraction: if (Array.isArray(value) && value.length > 10) continue A row whose `tags` array held ELEVEN entries therefore had that field skipped entirely — no posting, no error, no warning. The row then failed to match every filtered search on `tags`, including a query for a tag it demonstrably held, and the caller had no way to tell that from "no row matches". Eleven tags is not an exotic shape; the eleventh tag made the row invisible. Measured on the pin here: the where-clause returns [] on the base for all eleven values. The ceiling is not the defect. The silence was. THE LAW. MAX_INDEXED_ARRAY_LENGTH = 64, hardcoded (the zero-config law: no knob), sitting far above every legitimate multi-value field — tags, authors, categories, labels, participants — and far below any real embedding width, so the two populations do not overlap and nobody has to tune it. Arrays of scalars index in full up to the bound. Above it the WRITE IS REFUSED by name: MetadataArrayTooLargeError carries the field (its full dotted address), the length and the bound, and names the three cures. It fires at all four write doors — add, update, relate, updateRelation — beside the existing forged-system- key rejection, and walks nested bags because a nested field indexes under its dotted address exactly like a top-level one. THE ONE PLACE THE BOUND STILL SKIPS is a row already on disk, written by an older engine under the old rule and read back by a rebuild, a catch-up fold or a remove. extractIndexableFields serves all three, so refusing there would make an existing store un-rebuildable — the row is admitted and the skipped field is NARRATED with the field, the length and the bound. Never silent, either way. tests/integration/metadata-vector-exclusion.test.ts carried the old law as a green assertion ("should skip indexing large arrays (>10 elements)"). It is rewritten to the new one, plus a case proving a 64-element array indexes in full and its eleventh element is searchable. The original bug that suite exists for — per-dimension numeric field explosion — is still asserted on both paths. --- src/errors/brainyError.ts | 65 +++++ src/index.ts | 2 +- src/utils/metadataIndex.ts | 45 +++- src/utils/paramValidation.ts | 49 ++++ .../metadata-vector-exclusion.test.ts | 58 +++-- .../utils/metadataIndex-array-bound.test.ts | 242 ++++++++++++++++++ 6 files changed, 436 insertions(+), 25 deletions(-) create mode 100644 tests/unit/utils/metadataIndex-array-bound.test.ts diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index a58236e3..4301d3f7 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -405,3 +405,68 @@ export class MigrationInProgressError extends BrainyError { } } } + +/** + * THE INDEXABLE-ARRAY BOUND. An array-valued metadata field indexes one posting + * per element, so an unbounded array is an unbounded write — a 384-float + * embedding parked in the metadata bag would mint 384 postings for one row. + * The bound exists to keep that out of the index. + * + * 64 is hardcoded on purpose (the zero-config law: no knob). It sits far above + * every legitimate multi-value field the engine has seen — tags, authors, + * categories, labels, participant lists — and far below any real embedding + * width, so the two populations do not overlap and no caller has to tune it. + * + * It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array + * held eleven entries had that field skipped entirely and dropped out of every + * filtered search on it, with no error, no warning and no way to tell the + * difference from "no row matches". A rule this consequential is a law with a + * name and a refusal, not a `continue`. + */ +export const MAX_INDEXED_ARRAY_LENGTH = 64 + +/** + * A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}. + * + * Thrown at the WRITE door (`add` / `update` / `relate` / `updateRelation`), so + * the caller learns at the moment of writing that the field will not be + * searchable — rather than discovering it later as rows that quietly fail to + * match. Carries the field, its length and the bound so a handler can report + * or repair without parsing the message. + * + * The cure is one of: store the long array outside the indexed bag (`data` + * carries arbitrary content and is not indexed element-wise); pass an embedding + * as the first-class `vector` parameter, which is where a vector belongs; or + * shorten the field to the values that are actually queried. + */ +export class MetadataArrayTooLargeError extends BrainyError { + /** The metadata field whose array is too long (its full dotted address). */ + public readonly field: string + /** How many elements that array holds. */ + public readonly length: number + /** The bound it exceeded — {@link MAX_INDEXED_ARRAY_LENGTH}. */ + public readonly limit: number + + constructor(site: string, field: string, length: number, limit: number) { + super( + `${site}: metadata field '${field}' holds ${length} array elements, ` + + `over the ${limit}-element indexing bound. An array field indexes one ` + + `posting per element, so an unbounded array is an unbounded write. ` + + `This write is refused rather than indexed partially or skipped silently ` + + `— a skipped field drops the row out of every filtered search on '${field}' ` + + `with no way to tell that from "nothing matched". ` + + `Cures: put the long array in 'data' (stored, not indexed element-wise); ` + + `pass an embedding as the first-class 'vector' parameter; or keep only ` + + `the values you actually query in '${field}'.`, + 'VALIDATION', + false + ) + this.name = 'MetadataArrayTooLargeError' + this.field = field + this.length = length + this.limit = limit + if (Error.captureStackTrace) { + Error.captureStackTrace(this, MetadataArrayTooLargeError) + } + } +} diff --git a/src/index.ts b/src/index.ts index e946f15c..673e1e6f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -203,7 +203,7 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js // Base error + typed migration-lock error — thrown by any data-plane call while a // brain runs its one-time 7.x→8.0 upgrade; catch to answer HTTP 503 + Retry-After. -export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError } from './errors/brainyError.js' +export { BrainyError, MigrationInProgressError, GraphIndexNotReadyError, MetadataIndexNotReadyError, VectorIndexNotReadyError, ProtectedArtifactError, DerivedArtifactMissingError, MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from './errors/brainyError.js' export type { BrainyErrorType } from './errors/brainyError.js' // ============= 8.0 Db API — generational MVCC ============= diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 83d37379..1a882945 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -40,7 +40,7 @@ import { import { EntityIdMapper } from './entityIdMapper.js' import { RoaringBitmap32, roaringLibraryInitialize } from './roaring/index.js' import { FieldTypeInference, FieldType } from './fieldTypeInference.js' -import { BrainyError } from '../errors/brainyError.js' +import { BrainyError, MAX_INDEXED_ARRAY_LENGTH } from '../errors/brainyError.js' /** * Fields whose values are stored in the sparse index as BUCKETED values @@ -289,8 +289,10 @@ export class MetadataIndexManager implements MetadataIndexProvider { // No name-based exclude/allow lists — the field-addressing law: every // user field indexes, whatever its name ('content', 'data', 'id', // 'vector', … included). Bulk payloads are kept out by uniform value- - // SHAPE rules in extractIndexableFields (arrays >10 never become - // posting scalars; >100-char values index hashed), never by name. + // SHAPE rules in extractIndexableFields (arrays longer than + // MAX_INDEXED_ARRAY_LENGTH never become posting scalars, and the write + // door refuses them by name; >100-char values index hashed), never by + // field name. } // Initialize metadata cache with similar config to search cache @@ -1387,9 +1389,10 @@ export class MetadataIndexManager implements MetadataIndexProvider { * 'content', 'vector' in a bag are ordinary user fields) * - Record-frame plumbing (vector, connections, level, data, _rev, id) * never indexes — that is namespace routing, not a name carve-out - * - Value-SHAPE rules apply uniformly to all names: arrays >10 never - * become posting scalars; purely numeric key names (array indices) - * skip; >100-char values index hashed (normalizeValue) + * - Value-SHAPE rules apply uniformly to all names: arrays longer than + * MAX_INDEXED_ARRAY_LENGTH never become posting scalars (and say so — + * the write door refuses them outright); purely numeric key names + * (array indices) skip; >100-char values index hashed (normalizeValue) */ private extractIndexableFields(data: any): Array<{ field: string, value: any }> { const fields: Array<{ field: string, value: any }> = [] @@ -1451,13 +1454,37 @@ export class MetadataIndexManager implements MetadataIndexProvider { // This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...} if (/^\d+$/.test(key)) continue - // Skip large arrays (> 10 elements) - likely vectors or bulk data - if (Array.isArray(value) && value.length > 10) continue + // THE INDEXABLE-ARRAY BOUND ({@link MAX_INDEXED_ARRAY_LENGTH}). An + // array field mints one posting per element, so the index has always + // carried a ceiling — it was 10, and it was applied by this bare + // `continue`: an eleven-element `tags` array had its whole field + // skipped and the row dropped out of every filtered search on it, with + // no error, no warning, and nothing to distinguish that from "no row + // matches". The ceiling is not the defect; the silence was. + // + // The write door refuses this shape by name now + // (`MetadataArrayTooLargeError`, thrown from paramValidation's + // `rejectOversizeIndexArrays`), so a live add/update never reaches + // here over the bound. Reaching it means the row is ALREADY on disk — + // written by an older engine under the old rule — and this is a + // rebuild, a catch-up fold or a remove reading it back. Refusing there + // would make an existing store un-rebuildable, so the row is admitted + // and the skipped field is NARRATED instead. Never silent, either way. + if (Array.isArray(value) && value.length > MAX_INDEXED_ARRAY_LENGTH) { + prodLog.warn( + `[brainy] metadata field '${fullKey}' holds ${value.length} array elements, ` + + `over the ${MAX_INDEXED_ARRAY_LENGTH}-element indexing bound — the field is ` + + `NOT indexed for this row, so it will not match a where-clause on '${fullKey}'. ` + + `This row predates the bound (the write door refuses this shape now). ` + + `Move the long array into 'data', or pass an embedding as the 'vector' parameter.` + ) + continue + } if (value && typeof value === 'object' && !Array.isArray(value)) { // Recurse into nested objects (but not arrays), keeping the frame extract(value, fullKey, frame) - } else if (Array.isArray(value) && value.length <= 10) { + } else if (Array.isArray(value)) { // Small arrays: index as multi-value field (all with same field name) // Example: tags: ["javascript", "node"] → field="tags", value="javascript" + field="tags", value="node" for (const item of value) { diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 00790a4a..f1addb5b 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -18,6 +18,7 @@ import { findCallerLocation } from './callerLocation.js' import * as os from 'node:os' import * as fs from 'node:fs' import { parseFieldAddress, UnsupportedFindOptionError } from '../db/fieldAddressing.js' +import { MAX_INDEXED_ARRAY_LENGTH, MetadataArrayTooLargeError } from '../errors/brainyError.js' const getSystemMemory = (): number => { if (os) { @@ -538,8 +539,53 @@ function rejectForgedSystemKeys(metadata: Record | undefined, s } } +/** + * THE INDEXABLE-ARRAY BOUND, enforced at the write door. + * + * An array-valued metadata field indexes one posting per element, so the index + * has always carried a ceiling. It used to be 10, and it was applied by a bare + * `continue` deep inside field extraction: a row whose `tags` array held eleven + * entries had that field skipped entirely and dropped out of every filtered + * search on it — no error, no warning, and no way for the caller to tell the + * difference from "no row matches". Silence is the defect; the ceiling is not. + * + * The bound is now {@link MAX_INDEXED_ARRAY_LENGTH}, high enough that every + * legitimate multi-value field clears it, and it REFUSES here instead of + * dropping data downstream. Refusing at the write door is what makes it + * actionable: the caller learns at the moment of writing, with the field, the + * length and the bound in hand. + * + * Scope is the caller's own metadata bag — the values that become postings. + * Nested bags are walked, because a nested field indexes under its dotted + * address exactly like a top-level one. Arrays of OBJECTS are not walked: the + * index only ever makes postings from an array's scalar elements. + * + * @param metadata - The caller's metadata bag (undefined is fine). + * @param site - The write door's name, for the message ('add()', 'update()', …). + * @throws {MetadataArrayTooLargeError} Naming the field, its length and the bound. + */ +function rejectOversizeIndexArrays(metadata: Record | undefined, site: string): void { + if (!metadata) return + + const walk = (bag: Record, prefix: string): void => { + for (const [key, value] of Object.entries(bag)) { + const address = prefix ? `${prefix}.${key}` : key + if (Array.isArray(value)) { + if (value.length > MAX_INDEXED_ARRAY_LENGTH) { + throw new MetadataArrayTooLargeError(site, address, value.length, MAX_INDEXED_ARRAY_LENGTH) + } + } else if (value && typeof value === 'object') { + walk(value as Record, address) + } + } + } + + walk(metadata, '') +} + export function validateAddParams(params: AddParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'add()') + rejectOversizeIndexArrays(params.metadata as Record | undefined, 'add()') // 'data' is ABSENT only when null/undefined — an empty string ('') is real // content (a legitimate empty file's first write) and must not be treated // as missing. Falsy-but-present values (0, false, '') all count as present; @@ -608,6 +654,7 @@ export function validateAddParams(params: AddParams): void { */ export function validateUpdateParams(params: UpdateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'update()') + rejectOversizeIndexArrays(params.metadata as Record | undefined, 'update()') // Same absent-vs-empty distinction as validateAddParams: '' is a real new // value (e.g. truncating a file to empty content via overwrite), only // null/undefined means "no new data was given". @@ -682,6 +729,7 @@ export function validateUpdateParams(params: UpdateParams): void { */ export function validateRelateParams(params: RelateParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'relate()') + rejectOversizeIndexArrays(params.metadata as Record | undefined, 'relate()') // 8.0 verb-id contract (L.7): verb ids are UUIDs, generated by brainy. // RelateParams has no `id` field — an untyped caller passing one would // previously have it silently ignored (a generated UUID was used instead). @@ -731,6 +779,7 @@ export function validateRelateParams(params: RelateParams): void { */ export function validateUpdateRelationParams(params: UpdateRelationParams): void { rejectForgedSystemKeys(params.metadata as Record | undefined, 'updateRelation()') + rejectOversizeIndexArrays(params.metadata as Record | undefined, 'updateRelation()') if (!params.id) { throw new Error('id is required for updateRelation') } diff --git a/tests/integration/metadata-vector-exclusion.test.ts b/tests/integration/metadata-vector-exclusion.test.ts index 9e11f9dc..0ca25388 100644 --- a/tests/integration/metadata-vector-exclusion.test.ts +++ b/tests/integration/metadata-vector-exclusion.test.ts @@ -26,6 +26,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' import { existsSync, rmSync } from 'fs' +import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../src/errors/brainyError.js' describe('Metadata Vector Exclusion Fix', () => { let brainy: Brainy @@ -155,29 +156,56 @@ describe('Metadata Vector Exclusion Fix', () => { expect(results[0].entity.metadata?.name).toBe('Bob') }) - it('should skip indexing large arrays (>10 elements)', async () => { - // Add entity with a large array (not a vector, just bulk data). + it('should REFUSE an array over the indexing bound, by name', async () => { + // A large array (not a vector, just bulk data). This used to be SKIPPED in + // silence at a bound of 10 — the field simply vanished from the index and + // the row dropped out of every `where` on it, indistinguishably from "no + // row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES. const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`) - await brainy.add({ - type: NounType.Document, - data: 'Doc with large array', - metadata: { - name: 'Doc with large array', - items: largeArray - } - }) + const err = await brainy + .add({ + type: NounType.Document, + data: 'Doc with large array', + metadata: { + name: 'Doc with large array', + items: largeArray + } + }) + .catch((e: any) => e) - // Large arrays (> 10 elements) are deliberately skipped to avoid indexing - // bulk/vector-like payloads: 'items' must NOT appear, and the 100 elements - // must NOT have produced 100 indexed fields. + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('items') + expect(err.length).toBe(100) + expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) + + // Nothing was indexed from the refused write — no 'items' field, and above + // all no per-element numeric fields (the original explosion class). const fields = await brainy.getAvailableFields() expect(fields).not.toContain('items') const numericFields = fields.filter(f => /(^|\.)\d+$/.test(f)) expect(numericFields).toEqual([]) + }) - // The scalar 'name' field IS indexed. - expect(fields).toContain('name') + it('should index an array UP TO the bound — the old limit of 10 was the bug', async () => { + await brainy.add({ + type: NounType.Document, + data: 'Doc with a long-but-legitimate tag list', + metadata: { + name: 'Doc with many tags', + items: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`) + } + }) + + const fields = await brainy.getAvailableFields() + // The field IS indexed now, and still without per-element numeric fields. + expect(fields).toContain('items') + expect(fields.filter(f => /(^|\.)\d+$/.test(f))).toEqual([]) + + // And the eleventh element — the one the old bound silently dropped the + // whole field for — really is searchable. + const hits = await brainy.find({ where: { items: 'item10' } }) + expect(hits.length).toBeGreaterThan(0) }) it('should preserve HNSW vector search functionality', async () => { diff --git a/tests/unit/utils/metadataIndex-array-bound.test.ts b/tests/unit/utils/metadataIndex-array-bound.test.ts new file mode 100644 index 00000000..cbbf6b63 --- /dev/null +++ b/tests/unit/utils/metadataIndex-array-bound.test.ts @@ -0,0 +1,242 @@ +/** + * @module tests/unit/utils/metadataIndex-array-bound + * @description THE INDEXABLE-ARRAY BOUND — a law with a name and a refusal, + * not a `continue`. + * + * THE DEFECT. An array-valued metadata field indexes one posting per element, + * so the index has always carried a ceiling. It was 10, and it was applied by a + * bare `continue` deep inside field extraction: + * + * if (Array.isArray(value) && value.length > 10) continue + * + * A row whose `tags` array held ELEVEN entries therefore had that field skipped + * entirely — no posting, no error, no warning. The row then failed to match + * every filtered search on `tags`, including `{ tags: 'a-tag-it-really-has' }`, + * and the caller had no way to tell that from "no row matches". Eleven tags is + * not an exotic shape; the eleventh tag made the row invisible. + * + * THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH} = 64, + * hardcoded (the zero-config law: no knob), which clears every legitimate + * multi-value field and stays far below any embedding width. Above it the WRITE + * IS REFUSED by name — `MetadataArrayTooLargeError`, carrying the field, the + * length and the bound — at `add`, `update`, `relate` and `updateRelation` + * alike. Nothing is skipped in silence. + * + * THE ONE PLACE THE BOUND STILL SKIPS is a row already on disk, written by an + * older engine under the old rule and read back by a rebuild, a catch-up fold + * or a remove. Refusing there would make an existing store un-rebuildable — so + * the row is admitted and the skipped field is NARRATED. Both sides are pinned. + */ +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType, VerbType } from '../../../src/types/graphTypes' +import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError' +import { resolveEntityId } from '../../../src/utils/idNormalization' +import { prodLog } from '../../../src/utils/logger' + +/** `n` distinct scalar tags. */ +function tags(n: number, prefix = 't'): string[] { + return Array.from({ length: n }, (_, i) => `${prefix}${i}`) +} + +describe('the indexable-array bound', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brain.init() + }) + + describe('BELOW the bound: the array indexes, every element of it', () => { + it('the eleven-element array that used to vanish is searchable', async () => { + // ELEVEN — one over the old silent limit, the whole shape of the defect. + await brain.add({ + id: 'eleven', + data: 'a row with eleven tags', + type: NounType.Document, + metadata: { tags: tags(11) }, + vector: [] + }) + + // Every element is a posting, including the eleventh. + for (const tag of tags(11)) { + const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('eleven')) + } + }) + + it('indexes right up to the bound — all 64 elements', async () => { + await brain.add({ + id: 'at-bound', + data: 'a row at the bound', + type: NounType.Document, + metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) }, + vector: [] + }) + + // The first, the last, and one in the middle. + for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, 't31']) { + const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound')) + } + }) + + it('a nested bag\'s array indexes under its dotted address', async () => { + await brain.add({ + id: 'nested', + data: 'a row with a nested tag list', + type: NounType.Document, + metadata: { facets: { labels: tags(20, 'l') } }, + vector: [] + }) + const hits = await brain.find({ where: { 'facets.labels': 'l19' }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('nested')) + }) + }) + + describe('ABOVE the bound: the write is refused, by name', () => { + const OVER = MAX_INDEXED_ARRAY_LENGTH + 1 + + it('add() throws a typed error naming the field, the length and the bound', async () => { + const err = await brain + .add({ + id: 'too-many', + data: 'a row with too many tags', + type: NounType.Document, + metadata: { tags: tags(OVER) }, + vector: [] + } as any) + .catch((e: any) => e) + + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('tags') + expect(err.length).toBe(OVER) + expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) + expect(err.type).toBe('VALIDATION') + // The message carries all three, and names the cures. + expect(err.message).toContain('tags') + expect(err.message).toContain(String(OVER)) + expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) + expect(err.message).toContain('vector') + }) + + it('the refused row is not written at all — no half-indexed ghost', async () => { + await expect( + brain.add({ + id: 'refused', + data: 'refused', + type: NounType.Document, + metadata: { tags: tags(OVER) }, + vector: [] + } as any) + ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) + + expect(await brain.get('refused')).toBeNull() + const hits = await brain.find({ where: { tags: 't0' }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).not.toContain(resolveEntityId('refused')) + }) + + it('a 384-float embedding parked in the metadata bag is refused, not swallowed', async () => { + const err = await brain + .add({ + id: 'bag-vector', + data: 'an embedding in the wrong place', + type: NounType.Document, + metadata: { embedding: Array.from({ length: 384 }, (_, i) => i / 384) }, + vector: [] + } as any) + .catch((e: any) => e) + + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('embedding') + expect(err.length).toBe(384) + }) + + it('update() refuses it too', async () => { + await brain.add({ + id: 'grow', + data: 'starts small', + type: NounType.Document, + metadata: { tags: tags(3) }, + vector: [] + }) + await expect( + brain.update({ id: 'grow', metadata: { tags: tags(OVER) } } as any) + ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) + + // And the row keeps the values it had. + const hits = await brain.find({ where: { tags: 't1' }, limit: 10 } as any) + expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('grow')) + }) + + it('relate() refuses it on a verb\'s metadata', async () => { + await brain.add({ id: 'a', data: 'a', type: NounType.Thing, vector: [] }) + await brain.add({ id: 'b', data: 'b', type: NounType.Thing, vector: [] }) + await expect( + brain.relate({ + from: 'a', + to: 'b', + type: VerbType.RelatedTo, + metadata: { tags: tags(OVER) } + } as any) + ).rejects.toBeInstanceOf(MetadataArrayTooLargeError) + }) + + it('a nested oversize array is refused under its dotted address', async () => { + const err = await brain + .add({ + id: 'nested-over', + data: 'nested and too long', + type: NounType.Document, + metadata: { facets: { labels: tags(OVER, 'l') } }, + vector: [] + } as any) + .catch((e: any) => e) + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('facets.labels') + }) + }) + + describe('a row already on disk is admitted, and the skip is NARRATED', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('extraction over an old oversize row warns by field, length and bound', async () => { + const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + const index = (brain as any).metadataIndex + + // The shape an older engine persisted: the write door never saw it, so + // this reaches extraction directly — exactly as a rebuild or a remove + // reading the row back would. + const fields = index.extractIndexableFields({ + metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH + 5), keep: 'me' } + }) + + // The oversize field contributes nothing... + expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(0) + // ...the rest of the row indexes normally — the row is not rejected... + expect(fields.some((f: any) => f.field === 'keep' && f.value === 'me')).toBe(true) + // ...and the skip is said out loud, with everything needed to act on it. + expect(warn).toHaveBeenCalled() + const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n') + expect(said).toContain('tags') + expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH + 5)) + expect(said).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) + expect(said).toContain('NOT indexed') + }) + + it('an at-bound row on disk is indexed in full and says nothing', async () => { + const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) + const index = (brain as any).metadataIndex + + const fields = index.extractIndexableFields({ + metadata: { tags: tags(MAX_INDEXED_ARRAY_LENGTH) } + }) + expect(fields.filter((f: any) => f.field === 'tags')).toHaveLength(MAX_INDEXED_ARRAY_LENGTH) + + const said = warn.mock.calls.map((c: any[]) => String(c[0])).join('\n') + expect(said).not.toContain('indexing bound') + }) + }) +}) From f27a777615980cf7e69e660887bd329e2ddd7dee Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:48:25 -0700 Subject: [PATCH 129/150] fix(close): a read-only brain writes nothing under `_system/` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `readonly-close-no-marker` closed the clean-shutdown-marker half of this law and named the rest as a known residual. This is that residual, closed. MEASURED on the base: a read-only open → read → close rewrote FOUR files — `_system/__metadata_field_registry__.json.gz`, `type-statistics.json.gz`, `subtype-statistics.json.gz` and `verb-subtype-statistics.json.gz`. An IDLE reader that only opened and closed rewrote all four as well. The cause was not the closes the marker fix guarded. It was Phase 1 of closeDurableSteps, where every component flush ran unconditionally. A flush is a write by definition: MetadataIndexManager#flush() saves the field registry "even with no dirty fields" (its own comment), and the storage adapter's count flush re-stamps the three statistics files. A session that committed nothing re-stamped all four. Phase 2's closes were ungated too — the graph index's close drains both LSM MemTables to SSTables and stamps its watermark, and the optional vector/metadata `close` hooks (unimplemented in the reference engine, filled in by a native provider) persist buffered state. Every one of those calls now carries the same `!isReadOnly` guard the generation store already had. A reader still RELEASES what it holds, so Phase 2 is a branch rather than a skip: GraphAdjacencyIndex gains `stopBackgroundFlush()`, the non-writing half of its close, which clears the auto-flush interval that would otherwise outlive the session. `close()` now calls it too, so there is one place that owns the timer. Why this matters beyond tidiness: `_system/` is where a store keeps its evidence about itself — what the writer committed, what the projections have seen. A reader that rewrites any of it vouches for a state it only observed, and on shared or snapshot storage it mutates bytes another process owns. The pin hashes every file under `_system/` (and, in one case, the whole store) across a reader's open → read → close, names the four paths that used to move so a regression says which subsystem did it, and asserts the asymmetry holds in the other direction — a WRITER's close still persists. --- src/brainy.ts | 43 ++- src/graph/graphAdjacencyIndex.ts | 22 +- .../readonly-close-no-marker.test.ts | 12 +- .../readonly-close-writes-nothing.test.ts | 261 ++++++++++++++++++ 4 files changed, 322 insertions(+), 16 deletions(-) create mode 100644 tests/integration/readonly-close-writes-nothing.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index a97bdde2..5fd6c627 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -20834,34 +20834,45 @@ export class Brainy implements BrainyInterface { // Phase 1: Flush ALL components in parallel to persist buffered data // This is critical when cor native providers buffer data in Rust memory + // + // READ-ONLY GUARD, applied to EVERY flush here. A flush is a write by + // definition, and a reader has nothing of its own to persist — but these + // calls were not conditional, so a read-only open → read → close REWROTE + // four files under `_system/`: the metadata field registry (whose flush() + // saves it unconditionally, "even with no dirty fields"), and the three + // type/subtype statistics files the storage adapter's count flush stamps. + // Every one of them was re-stamped on a session that committed nothing. + // A reader must leave `_system/` exactly as it found it — the same law the + // clean-shutdown marker already lives under (see the generation-store + // guard below and `Brainy.openReadOnly`). await Promise.all([ // Flush HNSW dirty nodes (deferred persistence mode) (async () => { - if (this.index && typeof this.index.flush === 'function') { + if (this.index && !this.isReadOnly && typeof this.index.flush === 'function') { await this.index.flush() } })(), // Flush metadata index (field indexes + EntityIdMapper) (async () => { - if (this.metadataIndex && typeof this.metadataIndex.flush === 'function') { + if (this.metadataIndex && !this.isReadOnly && typeof this.metadataIndex.flush === 'function') { await this.metadataIndex.flush() } })(), // Flush graph adjacency index (LSM trees) (async () => { - if (this.graphIndex && typeof this.graphIndex.flush === 'function') { + if (this.graphIndex && !this.isReadOnly && typeof this.graphIndex.flush === 'function') { await this.graphIndex.flush() } })(), // Flush storage adapter counts (async () => { - if (this.storage && typeof this.storage.flushCounts === 'function') { + if (this.storage && !this.isReadOnly && typeof this.storage.flushCounts === 'function') { await this.storage.flushCounts() } })(), // Flush aggregation index state (async () => { - if (this._aggregationIndex) { + if (this._aggregationIndex && !this.isReadOnly) { await this._aggregationIndex.flush() } })(), @@ -20910,21 +20921,37 @@ export class Brainy implements BrainyInterface { // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 + // + // READ-ONLY GUARD, same law as Phase 1. Each of these closes is a WRITER: + // the graph index drains both LSM MemTables to SSTables and stamps its + // watermark, and the vector/metadata `close` hooks — optional doors the + // reference engine leaves unimplemented, but which a native provider fills + // in — persist their buffered state. None of that is a reader's to write. + // + // A reader still has to RELEASE what it holds, which is why this is a + // branch rather than a skip: `stopBackgroundFlush()` is the non-writing + // half of the graph index's close, clearing the auto-flush interval that + // would otherwise outlive the session. The optional hooks have no + // non-writing counterpart to call, and a provider that buffers nothing on + // a read-only open has nothing to release. await Promise.all([ (async () => { - if (this.graphIndex && typeof this.graphIndex.close === 'function') { + if (!this.graphIndex) return + if (this.isReadOnly) { + this.graphIndex.stopBackgroundFlush() + } else if (typeof this.graphIndex.close === 'function') { await this.graphIndex.close() } })(), (async () => { const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks - if (index && typeof index.close === 'function') { + if (index && !this.isReadOnly && typeof index.close === 'function') { await index.close() } })(), (async () => { const metadataIndex = this.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks - if (metadataIndex && typeof metadataIndex.close === 'function') { + if (metadataIndex && !this.isReadOnly && typeof metadataIndex.close === 'function') { await metadataIndex.close() } })(), diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index ebd3b90c..2c131a30 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1105,13 +1105,31 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { } /** - * Clean shutdown + * Stop the auto-flush interval WITHOUT writing anything. + * + * The non-writing half of {@link close}, for a shutdown that must leave the + * store byte-identical — a read-only brain's close. `close()` itself is a + * writer: it drains both LSM MemTables to SSTables and stamps the watermark, + * which is exactly right for a writer and forbidden for a reader. A reader + * still has to release this interval, though: it is the one piece of this + * index that outlives the close and could fire against a store the session no + * longer owns. + * + * @returns Nothing. */ - async close(): Promise { + stopBackgroundFlush(): void { if (this.flushTimer) { clearInterval(this.flushTimer) this.flushTimer = undefined } + } + + /** + * Clean shutdown — drains both trees and stamps the watermark. THIS WRITES; + * a read-only brain must call {@link stopBackgroundFlush} instead. + */ + async close(): Promise { + this.stopBackgroundFlush() // Close both LSM-trees (will flush MemTables to SSTables) if (this.initialized) { diff --git a/tests/integration/readonly-close-no-marker.test.ts b/tests/integration/readonly-close-no-marker.test.ts index 7bcf99df..ad9357db 100644 --- a/tests/integration/readonly-close-no-marker.test.ts +++ b/tests/integration/readonly-close-no-marker.test.ts @@ -149,12 +149,12 @@ describe('a read-only brain writes no clean-shutdown evidence', () => { brain = null // The FILE SET under `_system/` is unchanged — a reader creates and - // removes nothing. (Other files under `_system/` — e.g. the metadata - // field registry, which stamps its own `lastUpdated` on every persist — - // are a pre-existing, separate concern outside this fix's scope: this - // pin is specifically about the generation store's clean-shutdown - // evidence, not about every subsystem's close() being a true no-op for - // a reader.) + // removes nothing. This pin is specifically about the generation store's + // clean-shutdown evidence. The wider law — that a reader leaves EVERY + // file under `_system/` byte-identical, which this fix left open as a + // known residual (the metadata field registry and the three statistics + // files were still re-stamped by a reader's close) — is closed and pinned + // in `readonly-close-writes-nothing.test.ts`. const after = snapshotDir(systemDir()) expect([...after.keys()].sort()).toEqual([...before.keys()].sort()) diff --git a/tests/integration/readonly-close-writes-nothing.test.ts b/tests/integration/readonly-close-writes-nothing.test.ts new file mode 100644 index 00000000..701a1974 --- /dev/null +++ b/tests/integration/readonly-close-writes-nothing.test.ts @@ -0,0 +1,261 @@ +/** + * @module tests/integration/readonly-close-writes-nothing + * @description A READ-ONLY BRAIN LEAVES `_system/` BYTE-IDENTICAL — the WHOLE + * directory, not just the clean-shutdown marker. + * + * `readonly-close-no-marker` closed the marker half of this law and named the + * rest as a known, out-of-scope residual: + * + * "Other files under `_system/` — e.g. the metadata field registry, which + * stamps its own `lastUpdated` on every persist — are a pre-existing, + * separate concern outside this fix's scope." + * + * This is that residual, closed. MEASURED on the base before the fix, a + * read-only open → read → close rewrote FOUR files: + * + * _system/__metadata_field_registry__.json.gz + * _system/type-statistics.json.gz + * _system/subtype-statistics.json.gz + * _system/verb-subtype-statistics.json.gz + * + * THE CAUSE was not the closes the marker fix guarded — it was Phase 1 of + * `closeDurableSteps`, where every component flush ran unconditionally. A flush + * is a write by definition: `MetadataIndexManager#flush()` saves the field + * registry "even with no dirty fields" (its own comment), and the storage + * adapter's count flush re-stamps the three statistics files. A session that + * committed nothing re-stamped all four. Phase 2's closes were ungated too — + * the graph index's close drains both LSM MemTables and stamps a watermark, + * and the optional vector/metadata `close` hooks (unimplemented in the + * reference engine, filled in by a native provider) persist buffered state. + * + * THE LAW. A reader writes nothing, anywhere under `_system/`, at open or at + * close. It still RELEASES what it holds: the graph index's auto-flush interval + * is cleared through `stopBackgroundFlush()`, the non-writing half of its + * close, so nothing outlives the session. + * + * WHY IT MATTERS beyond tidiness: `_system/` is where a store keeps its + * evidence about itself — what the writer committed, what the projections have + * seen. A reader that rewrites any of it is vouching for a state it only + * observed, and on shared or snapshot storage it mutates bytes another process + * owns. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtempSync, rmSync, readdirSync, readFileSync, statSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' + +/** Recursively hash every regular file under `dir`, keyed by its path relative to `dir`. */ +function snapshotDir(dir: string): Map { + const out = new Map() + const walk = (rel: string): void => { + const abs = rel ? join(dir, rel) : dir + let entries: string[] + try { + entries = readdirSync(abs) + } catch { + return + } + for (const name of entries) { + const childRel = rel ? join(rel, name) : name + const childAbs = join(dir, childRel) + const st = statSync(childAbs) + if (st.isDirectory()) { + walk(childRel) + } else if (st.isFile()) { + out.set(childRel, createHash('sha256').update(readFileSync(childAbs)).digest('hex')) + } + } + } + walk('') + return out +} + +/** Every path where `after` differs from `before`, labelled — the failure message. */ +function diff(before: Map, after: Map): string[] { + const lines: string[] = [] + for (const [path, hash] of after) { + if (!before.has(path)) lines.push(`ADDED ${path}`) + else if (before.get(path) !== hash) lines.push(`CHANGED ${path}`) + } + for (const path of before.keys()) if (!after.has(path)) lines.push(`REMOVED ${path}`) + return lines.sort() +} + +describe('a read-only brain writes nothing under `_system/`', () => { + let dir: string + let brain: Brainy | null = null + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'brainy-readonly-writes-')) + }) + + afterEach(async () => { + if (brain) { + try { + await brain.close() + } catch { + /* already closed */ + } + brain = null + } + try { + rmSync(dir, { recursive: true, force: true }) + } catch { + /* ignore */ + } + }) + + const systemDir = () => join(dir, '_system') + + /** + * A writer seeds a store with nouns, verbs and queryable metadata — enough + * that the field registry, the statistics files and the graph index all hold + * real content — then closes cleanly. + */ + async function seedStore(): Promise { + const writer = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir } + }) + await writer.init() + for (let i = 0; i < 6; i++) { + await writer.add({ + id: `seed-${i}`, + data: `seed entity ${i}`, + type: i % 2 === 0 ? NounType.Concept : NounType.Document, + metadata: { lane: i % 2 === 0 ? 'alpha' : 'beta', rank: i, tags: [`t${i}`, 'shared'] }, + vector: [] + }) + } + for (let i = 1; i < 6; i++) { + await writer.relate({ from: 'seed-0', to: `seed-${i}`, type: VerbType.RelatedTo }) + } + await writer.flush() + await writer.close() + } + + it('open → read → close leaves every file under `_system/` byte-identical', async () => { + await seedStore() + + const before = snapshotDir(systemDir()) + expect(before.size, 'the writer left a populated `_system/`').toBeGreaterThan(0) + + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + expect(brain.isReadOnly).toBe(true) + + // Exercise the read surface that drives each subsystem: statistics (counts), + // a metadata filter (field index + registry), a graph walk (adjacency), a + // vector search, and a direct get. + await brain.stats() + await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) + await brain.find({ where: { tags: 'shared' }, limit: 10 } as any) + await brain.find({ connected: { from: 'seed-0', direction: 'out' }, limit: 10 } as any) + await brain.get('seed-1') + + await brain.close() + brain = null + + const after = snapshotDir(systemDir()) + const changes = diff(before, after) + expect(changes, `a reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) + }, 120_000) + + it('names the four files that used to change — the measured shape of the defect', async () => { + await seedStore() + const before = snapshotDir(systemDir()) + + // These are the exact paths the base rewrote. Naming them keeps the pin + // honest about what it caught: if a future change reintroduces the write, + // the test above fails and this one says which subsystem did it. + const previouslyRewritten = [ + '__metadata_field_registry__.json.gz', + 'type-statistics.json.gz', + 'subtype-statistics.json.gz', + 'verb-subtype-statistics.json.gz' + ] + for (const name of previouslyRewritten) { + expect(before.has(name), `fixture must contain ${name}`).toBe(true) + } + + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await brain.stats() + await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) + await brain.close() + brain = null + + const after = snapshotDir(systemDir()) + for (const name of previouslyRewritten) { + expect(after.get(name), `${name} was rewritten by a reader`).toBe(before.get(name)) + } + }, 120_000) + + it('a reader that only opens and closes — touching nothing — writes nothing', async () => { + await seedStore() + const before = snapshotDir(systemDir()) + + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await brain.close() + brain = null + + const changes = diff(before, snapshotDir(systemDir())) + expect(changes, `an idle reader modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) + }, 120_000) + + it('two readers in sequence each leave the store exactly as they found it', async () => { + await seedStore() + const before = snapshotDir(systemDir()) + + for (let i = 0; i < 2; i++) { + const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await reader.find({ where: { lane: 'beta' }, limit: 10 } as any) + await reader.close() + const changes = diff(before, snapshotDir(systemDir())) + expect(changes, `reader ${i + 1} modified \`_system/\`:\n${changes.join('\n')}`).toEqual([]) + } + }, 120_000) + + it('the store outside `_system/` is untouched too — a reader writes nowhere', async () => { + await seedStore() + const before = snapshotDir(dir) + + brain = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: dir } }) + await brain.stats() + await brain.find({ where: { lane: 'alpha' }, limit: 10 } as any) + await brain.close() + brain = null + + const changes = diff(before, snapshotDir(dir)) + expect(changes, `a reader modified the store:\n${changes.join('\n')}`).toEqual([]) + }, 120_000) + + it('a WRITER still persists on close — the guard did not disarm the write path', async () => { + await seedStore() + const before = snapshotDir(systemDir()) + + const writer = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await writer.init() + await writer.add({ + id: 'after-reader', + data: 'a new row', + type: NounType.Concept, + metadata: { lane: 'gamma', rank: 99 }, + vector: [] + }) + await writer.close() + + // The writer's close DID move `_system/` — that is the whole point of the + // asymmetry, and the guard must not have flattened it. + expect(diff(before, snapshotDir(systemDir())).length).toBeGreaterThan(0) + + // And the row is really there on the next open. + const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + await reopened.init() + brain = reopened + const hits = await reopened.find({ where: { lane: 'gamma' }, limit: 10 } as any) + expect(hits.length).toBe(1) + }, 120_000) +}) From 72c8ee6acd920069885f0f6f00a6d68d083d36a2 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 13:56:57 -0700 Subject: [PATCH 130/150] =?UTF-8?q?fix(contract):=20the=20flush=20gate's?= =?UTF-8?q?=20internals=20are=20#-private=20=E2=80=94=20they=20are=20not?= =?UTF-8?q?=20doors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 10.4.11 flush single-flight work added `startFlushLeader` and `promoteQueuedFlush` as TypeScript `private` methods. `private` is erased at compile time, so both still land on the prototype — and the contract manifest emitter reads the surface the BUILD exposes, skipping only names that start with an underscore. On the next regeneration both would have been emitted as contract doors, obliging every engine implementing contract 1 to provide the flush gate's own bookkeeping. A door is a promise; these are internals. Converted to ECMAScript-private (`#`), which keeps them off the prototype entirely, and the reason is recorded on both so the next internal is not written as `private` by habit. `_runFlush` — the flush body itself — was already safe by the emitter's underscore rule. Verified: `npm run build && node scripts/emit-contract-manifest.mjs` then `--check` green at 302 doors, with neither name present. TWO MANIFEST NOTES, both deliberate and neither hidden: 1. The regenerated manifest gains `MetadataArrayTooLargeError`. The emitter lists every `*Error` export from brainyError.js, and that class is the write door's refusal for an over-bound metadata array (this branch's array-bound commit). It is a real addition to the engine's error surface, so the manifest is right to carry it — flagged here because it is a contract-surface change that the cut should accept knowingly, not a side effect that slipped in. 2. `armIdleFlushTimer` and `kickBackgroundFlush` are TypeScript `private` in src and ARE already in the committed manifest as doors — the same leak, one release older. They are left exactly as they are: removing a name the manifest already publishes is a contract deletion, not a hygiene fix, and it belongs to whoever owns contract 1 rather than to this branch. --- docs/api-contract.json | 1 + src/brainy.ts | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/api-contract.json b/docs/api-contract.json index 12cb37c8..c4f4e056 100644 --- a/docs/api-contract.json +++ b/docs/api-contract.json @@ -1507,6 +1507,7 @@ "BrainyError", "DerivedArtifactMissingError", "GraphIndexNotReadyError", + "MetadataArrayTooLargeError", "MetadataIndexNotReadyError", "MigrationInProgressError", "ProtectedArtifactError", diff --git a/src/brainy.ts b/src/brainy.ts index 5fd6c627..15f51ef9 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -13342,25 +13342,33 @@ export class Brainy implements BrainyInterface { } return this._flushQueued } - return this.startFlushLeader() + return this.#startFlushLeader() } /** * @description Run one flush body as the leader and install it as - * `_flushInFlight`. On settle — resolved OR rejected — the gate opens and + * `_flushInFlight`. + * + * ECMAScript-private (`#`), not TypeScript `private`: `private` is erased at + * compile time, so the method still lands on the prototype and the contract + * manifest — which reads the surface the BUILD exposes — emitted it as a + * door. A door is a promise every engine implementing the contract must + * keep, and this is the flush gate's own bookkeeping, not a promise. `#` + * keeps it off the prototype, so the emitter cannot see it. + * On settle — resolved OR rejected — the gate opens and * the ONE queued waiter (if any) is promoted. The `finally` callback returns * nothing on purpose: a callback that returned the promoted run's promise * would make the leader await its own follower. * @returns The leader's own promise, settling on its own body alone. */ - private startFlushLeader(): Promise { + #startFlushLeader(): Promise { const run = this._runFlush() // `finally` and not `then`: a failed flush must still open the gate, or // one rejection would wedge every later flush behind a promise nobody // will ever settle. const gated: Promise = run.finally(() => { if (this._flushInFlight === gated) this._flushInFlight = null - this.promoteQueuedFlush() + this.#promoteQueuedFlush() }) this._flushInFlight = gated return gated @@ -13368,12 +13376,14 @@ export class Brainy implements BrainyInterface { /** * @description Promote the single queued waiter (if one is waiting) to - * leader and settle its deferred from that run. Never throws into the + * leader and settle its deferred from that run. ECMAScript-private for the + * same reason as {@link flush}'s leader starter: internals are not doors. + * Never throws into the * leader's `finally`: a synchronous failure starting the promoted run is * reported to the waiter, which must be settled on every path. * @returns Nothing. */ - private promoteQueuedFlush(): void { + #promoteQueuedFlush(): void { const settle = this._flushQueuedSettle if (!settle) return // Clear BEFORE starting, so the promoted run's own joiners queue afresh @@ -13381,7 +13391,7 @@ export class Brainy implements BrainyInterface { this._flushQueued = null this._flushQueuedSettle = null try { - this.startFlushLeader().then(settle.resolve, settle.reject) + this.#startFlushLeader().then(settle.resolve, settle.reject) } catch (error) { settle.reject(error) } From e49a73e52945eef72ee533564c8f7aa971b0202d Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:11:27 -0700 Subject: [PATCH 131/150] docs(flush): the ES-private note reads after the gate's contract, not through it The #-private rationale landed spliced into the middle of each method's description, cutting one sentence in half. Same words, moved below the behaviour they annotate. --- src/brainy.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index 15f51ef9..b8eb7f56 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -13347,18 +13347,17 @@ export class Brainy implements BrainyInterface { /** * @description Run one flush body as the leader and install it as - * `_flushInFlight`. - * - * ECMAScript-private (`#`), not TypeScript `private`: `private` is erased at - * compile time, so the method still lands on the prototype and the contract - * manifest — which reads the surface the BUILD exposes — emitted it as a - * door. A door is a promise every engine implementing the contract must - * keep, and this is the flush gate's own bookkeeping, not a promise. `#` - * keeps it off the prototype, so the emitter cannot see it. - * On settle — resolved OR rejected — the gate opens and + * `_flushInFlight`. On settle — resolved OR rejected — the gate opens and * the ONE queued waiter (if any) is promoted. The `finally` callback returns * nothing on purpose: a callback that returned the promoted run's promise * would make the leader await its own follower. + * + * ECMAScript-private (`#`), not TypeScript `private`: `private` is erased at + * compile time, so the method would still land on the prototype — and the + * contract manifest reads the surface the BUILD exposes, so it would emit + * this as a contract door. A door is a promise every engine implementing the + * contract must keep; this is the flush gate's own bookkeeping. `#` keeps it + * off the prototype, where the emitter cannot see it. * @returns The leader's own promise, settling on its own body alone. */ #startFlushLeader(): Promise { @@ -13376,11 +13375,12 @@ export class Brainy implements BrainyInterface { /** * @description Promote the single queued waiter (if one is waiting) to - * leader and settle its deferred from that run. ECMAScript-private for the - * same reason as {@link flush}'s leader starter: internals are not doors. - * Never throws into the + * leader and settle its deferred from that run. Never throws into the * leader's `finally`: a synchronous failure starting the promoted run is * reported to the waiter, which must be settled on every path. + * + * ECMAScript-private for the same reason as the leader starter above: + * internals are not doors. * @returns Nothing. */ #promoteQueuedFlush(): void { From a2820e81afc629b76a3d842e9331902a082df70f Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:15:39 -0700 Subject: [PATCH 132/150] =?UTF-8?q?ci(release):=20mechanize=20the=20releas?= =?UTF-8?q?es-wall=20entry=20=E2=80=94=20never=20hand-written=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every release used to get its releases/open-brainy.json entry typed by hand after the fact. scripts/wall-entry.mjs derives it from the CHANGELOG entry release.sh just composed (headline = first bullet, items = every bullet, hash stripped) and prepends it, refusing by name on a duplicate version and validating the whole file's shape + newest-first ordering before and after it writes. release.sh now runs it as its own step, between the CHANGELOG update and the release commit, and stages releases/open-brainy.json into that commit. The product engine's rail runs this identical script against its own releases/brainy.json, unchanged — each repo's wall file lives beside the CHANGELOG it derives from; there is no cross-repo step. A --check mode validates a wall file's exact key set, field types, and newest-first ordering with no duplicates, read-only. tests/unit/release/wall-entry.test.ts covers derivation, prepend, duplicate refusal, and --check's shape/ordering checks over temp copies — never the real files. --check also runs green against both releases/open-brainy.json and releases/brainy.json as they stand today. --- scripts/release.sh | 13 +- scripts/wall-entry.mjs | 364 ++++++++++++++++++++++++++ tests/unit/release/wall-entry.test.ts | 216 +++++++++++++++ 3 files changed, 591 insertions(+), 2 deletions(-) create mode 100644 scripts/wall-entry.mjs create mode 100644 tests/unit/release/wall-entry.test.ts diff --git a/scripts/release.sh b/scripts/release.sh index 5d434320..07d225ce 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -154,7 +154,8 @@ else fi # Create new changelog entry -CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) ($(date +%Y-%m-%d)) +RELEASE_DATE=$(date +%Y-%m-%d) +CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v${CURRENT_VERSION}...v${NEW_VERSION}) (${RELEASE_DATE}) ${COMMITS} " @@ -174,9 +175,17 @@ if [ -f "CHANGELOG.md" ]; then fi echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" +# Step 6b: Update the releases wall entry — mechanical, derived from the +# CHANGELOG entry just composed. The fleet's HQ page reads releases/open-brainy.json +# directly; this used to be hand-written after every release (David: never +# again — make it a step of the rail). +echo -e "${BLUE}5️⃣▸ Updating the releases wall...${NC}" +node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md +echo -e "${GREEN}✅ Releases wall updated${NC}\n" + # Step 7: Create release commit echo -e "${BLUE}6️⃣ Creating release commit...${NC}" -git add package.json package-lock.json CHANGELOG.md +git add package.json package-lock.json CHANGELOG.md releases/open-brainy.json git commit -m "chore(release): ${NEW_VERSION}" echo -e "${GREEN}✅ Release commit created${NC}\n" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs new file mode 100644 index 00000000..998431da --- /dev/null +++ b/scripts/wall-entry.mjs @@ -0,0 +1,364 @@ +#!/usr/bin/env node +/** + * @module scripts/wall-entry + * @description The releases-wall entry, made mechanical. The fleet's HQ page + * reads one public JSON per product (releases/.json — shape + * {product, entries:[{version, date, headline, items, url, thumb}], history}). + * Those entries were hand-written after every release; this script is the + * one door that composes one, so it never has to be typed by hand again. + * + * Two modes: + * + * 1. Generate + write in place (default): + * node wall-entry.mjs --product

--version --date \ + * --from-changelog [--file releases/

.json] + * Derives an entry from the CHANGELOG.md entry for (headline = the + * entry's first bullet, items = every bullet, trimmed of its trailing + * commit hash), prepends it to --file (default releases/.json, + * newest first), refusing by name if is already present, and + * validates the whole file's shape + ordering before and after writing. + * Both engines run this identically, each against its own repo's + * releases/.json — the wall file always lives beside the + * CHANGELOG it is derived from, never in another repo. + * + * 2. Validate only (--check): + * node wall-entry.mjs --check --file + * Validates the file's exact key set (top-level and per-entry), field + * types, and strict-descending semver ordering with no duplicates. + * Read-only; never writes. Exit 0 = clean, exit 1 = named violations + * printed to stderr. + * + * No dependencies — CHANGELOG parsing, semver comparison, and JSON shape + * checking are all hand-rolled below. + */ + +import { readFileSync, writeFileSync, existsSync } from 'node:fs' + +const ENTRY_KEYS = ['version', 'date', 'headline', 'items', 'url', 'thumb'] +const FILE_KEYS = ['product', 'entries', 'history'] + +// The public release-page URL pattern, by product — only products with a +// PUBLIC forge repo get a derived link. A product without an entry here +// (e.g. "brainy", whose repo is private) gets url: null, matching every +// entry the fleet has shipped for it so far — a private link would 404 for +// anyone reading the public HQ page. +const RELEASE_URL_PATTERNS = { + 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, +} + +/** + * Parse argv into a flag map. `--flag value` sets a string; `--flag` alone + * (end of argv, or followed by another `--flag`) sets boolean true. + * @param {string[]} argv + * @returns {Record} + */ +function parseArgs(argv) { + /** @type {Record} */ + const args = {} + for (let i = 0; i < argv.length; i++) { + const a = argv[i] + if (!a.startsWith('--')) continue + const key = a.slice(2) + const next = argv[i + 1] + if (next === undefined || next.startsWith('--')) { + args[key] = true + } else { + args[key] = next + i++ + } + } + return args +} + +/** + * Print a loud, named error and exit 1. Every refusal in this script goes + * through here so the failure mode is always the same shape: "wall-entry: ". + * @param {string} message + * @returns {never} + */ +function fail(message) { + console.error(`wall-entry: ${message}`) + process.exit(1) +} + +/** + * @param {string} version + * @returns {{major: number, minor: number, patch: number, pre: string | null} | null} + */ +function parseSemver(version) { + const m = /^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(version) + if (!m) return null + return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]), pre: m[4] ?? null } +} + +/** + * @param {string} a + * @param {string} b + * @returns {number} positive if a > b, negative if a < b, 0 if equal. + */ +function compareSemver(a, b) { + const pa = parseSemver(a) + const pb = parseSemver(b) + if (!pa || !pb) throw new Error(`cannot compare non-semver versions "${a}" vs "${b}"`) + if (pa.major !== pb.major) return pa.major - pb.major + if (pa.minor !== pb.minor) return pa.minor - pb.minor + if (pa.patch !== pb.patch) return pa.patch - pb.patch + if (pa.pre === pb.pre) return 0 + if (pa.pre === null) return 1 // a release outranks any prerelease of the same core version + if (pb.pre === null) return -1 + return pa.pre < pb.pre ? -1 : pa.pre > pb.pre ? 1 : 0 +} + +/** + * Validate a wall file's full shape: top-level keys, per-entry keys and + * field types, and strict-descending semver ordering with no duplicates. + * Collects every violation instead of failing on the first, so --check + * reports the whole picture in one pass. + * @param {unknown} data + * @returns {string[]} Violation messages; empty means the file is clean. + */ +function validateShape(data) { + /** @type {string[]} */ + const errors = [] + + if (typeof data !== 'object' || data === null || Array.isArray(data)) { + return ['top level: expected a JSON object'] + } + const obj = /** @type {Record} */ (data) + + const topKeys = Object.keys(obj) + const missingTop = FILE_KEYS.filter((k) => !(k in obj)) + const extraTop = topKeys.filter((k) => !FILE_KEYS.includes(k)) + if (missingTop.length) errors.push(`top level: missing key(s) ${missingTop.join(', ')}`) + if (extraTop.length) errors.push(`top level: unexpected key(s) ${extraTop.join(', ')}`) + + if (typeof obj.product !== 'string' || obj.product.trim() === '') { + errors.push('top level: "product" must be a non-empty string') + } + if (typeof obj.history !== 'string' || obj.history.trim() === '') { + errors.push('top level: "history" must be a non-empty string') + } + if (!Array.isArray(obj.entries)) { + errors.push('top level: "entries" must be an array') + return errors // nothing further to check without an array + } + + const entries = /** @type {unknown[]} */ (obj.entries) + entries.forEach((rawEntry, i) => { + const label = `entries[${i}]` + if (typeof rawEntry !== 'object' || rawEntry === null || Array.isArray(rawEntry)) { + errors.push(`${label}: expected an object`) + return + } + const entry = /** @type {Record} */ (rawEntry) + const keys = Object.keys(entry) + const missing = ENTRY_KEYS.filter((k) => !(k in entry)) + const extra = keys.filter((k) => !ENTRY_KEYS.includes(k)) + if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) + if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) + + if (typeof entry.version !== 'string' || !parseSemver(entry.version)) { + errors.push(`${label}: "version" must be a semver string (got ${JSON.stringify(entry.version)})`) + } + if (typeof entry.date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(entry.date) || Number.isNaN(Date.parse(entry.date))) { + errors.push(`${label}: "date" must be a YYYY-MM-DD string (got ${JSON.stringify(entry.date)})`) + } + if (typeof entry.headline !== 'string' || entry.headline.trim() === '') { + errors.push(`${label}: "headline" must be a non-empty string`) + } + if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { + errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) + } + if (!(entry.url === null || typeof entry.url === 'string')) { + errors.push(`${label}: "url" must be a string or null`) + } + if (!(entry.thumb === null || typeof entry.thumb === 'string')) { + errors.push(`${label}: "thumb" must be a string or null`) + } + }) + + // Ordering: newest first, strictly descending, no duplicate versions — + // checked only over entries whose version parsed (a bad version is + // already reported above; comparing it too would just be noise). + const versioned = entries + .map((e, i) => ({ i, version: /** @type {any} */ (e)?.version })) + .filter((e) => typeof e.version === 'string' && parseSemver(e.version)) + for (let i = 0; i < versioned.length - 1; i++) { + const a = versioned[i] + const b = versioned[i + 1] + const cmp = compareSemver(a.version, b.version) + if (cmp === 0) { + errors.push(`entries[${a.i}] and entries[${b.i}]: duplicate version ${a.version}`) + } else if (cmp < 0) { + errors.push(`entries[${a.i}] (${a.version}) sits above entries[${b.i}] (${b.version}) — not newest-first`) + } + } + + return errors +} + +/** + * Extract one version's entry body from a standard-version-style CHANGELOG.md + * (headings `### [version](url) (date)`, followed by `- bullet (hash)` lines + * until the next heading or EOF). + * @param {string} changelog + * @param {string} version + * @returns {string[]} Bullet lines, trimmed of their leading "- " and + * trailing " (hash)". + */ +function extractChangelogBullets(changelog, version) { + const lines = changelog.split('\n') + const headingRe = /^### \[([^\]]+)\]\(.*\)\s*\(\d{4}-\d{2}-\d{2}\)\s*$/ + let start = -1 + for (let i = 0; i < lines.length; i++) { + const m = headingRe.exec(lines[i]) + if (m && m[1] === version) { + start = i + 1 + break + } + } + if (start === -1) { + fail( + `version ${version} has no CHANGELOG entry yet — run this after the CHANGELOG step composes "### [${version}]", not before`, + ) + } + /** @type {string[]} */ + const bullets = [] + for (let i = start; i < lines.length; i++) { + if (headingRe.test(lines[i])) break // next entry starts + const bulletMatch = /^- (.+?)(?:\s\(([0-9a-f]{6,40})\))?$/.exec(lines[i].trim()) + if (lines[i].trim().startsWith('- ') && bulletMatch) { + const text = bulletMatch[1].trim() + if (text) bullets.push(text) + } + } + if (bullets.length === 0) { + fail(`version ${version}'s CHANGELOG entry has no bullets to derive a headline/items from`) + } + return bullets +} + +/** + * Derive a wall entry from a CHANGELOG.md. + * @param {{product: string, version: string, date: string, changelogPath: string, url?: string | null, thumb?: string | null}} opts + * @returns {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} + */ +function deriveEntry({ product, version, date, changelogPath, url, thumb }) { + if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) + if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || Number.isNaN(Date.parse(date))) { + fail(`--date "${date}" is not a YYYY-MM-DD date`) + } + if (!existsSync(changelogPath)) fail(`--from-changelog "${changelogPath}" does not exist`) + + const changelog = readFileSync(changelogPath, 'utf8') + const items = extractChangelogBullets(changelog, version) + const headline = items[0] + + const resolvedUrl = url !== undefined ? url : (RELEASE_URL_PATTERNS[product]?.(version) ?? null) + const resolvedThumb = thumb !== undefined ? thumb : null + + return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } +} + +/** + * Load and shape-validate a wall file. + * @param {string} filePath + * @returns {Record} + */ +function loadWallFile(filePath) { + if (!existsSync(filePath)) fail(`--file "${filePath}" does not exist`) + /** @type {unknown} */ + let data + try { + data = JSON.parse(readFileSync(filePath, 'utf8')) + } catch (err) { + fail(`--file "${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) + } + const errors = validateShape(data) + if (errors.length) { + fail(`--file "${filePath}" fails shape validation before any write —\n ${errors.join('\n ')}`) + } + return /** @type {Record} */ (data) +} + +/** + * Prepend `entry` to the wall file at `filePath`, refusing by name if the + * version is already present, validating before and after, and writing the + * file back with the repo's exact formatting (2-space JSON, trailing newline). + * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {string} filePath + * @param {string | undefined} expectedProduct + */ +function applyEntry(entry, filePath, expectedProduct) { + const wall = loadWallFile(filePath) + + if (expectedProduct && wall.product !== expectedProduct) { + fail( + `--file "${filePath}" has product "${wall.product}", but --product "${expectedProduct}" was given — refusing a cross-product write`, + ) + } + + if (wall.entries.some((e) => e.version === entry.version)) { + fail(`refusing — version ${entry.version} is already present in "${filePath}"`) + } + + wall.entries = [entry, ...wall.entries] + + const postErrors = validateShape(wall) + if (postErrors.length) { + fail(`the entry for ${entry.version} would leave "${filePath}" invalid —\n ${postErrors.join('\n ')}`) + } + + writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') + console.log(`wall-entry: wrote v${entry.version} to "${filePath}" (${wall.entries.length} entries, newest first)`) +} + +function main() { + const args = parseArgs(process.argv.slice(2)) + + if (args.check) { + const filePath = /** @type {string | undefined} */ (args.file) ?? + (typeof args.product === 'string' ? `releases/${args.product}.json` : undefined) + if (!filePath) fail('--check needs --file (or --product to default to releases/.json)') + const wall = loadWallFile(/** @type {string} */ (filePath)) + console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) + process.exit(0) + } + + // Generate mode (default): --product, --version, --date, --from-changelog required. + const product = /** @type {string | undefined} */ (args.product) + const version = /** @type {string | undefined} */ (args.version) + const date = /** @type {string | undefined} */ (args.date) + const fromChangelog = /** @type {string | undefined} */ (args['from-changelog']) + + const missing = [] + if (!product) missing.push('--product') + if (!version) missing.push('--version') + if (!date) missing.push('--date') + if (!fromChangelog) missing.push('--from-changelog') + if (missing.length) { + fail( + `missing required flag(s): ${missing.join(', ')}\n` + + 'Usage:\n' + + ' wall-entry.mjs --product

--version --date --from-changelog [--file releases/

.json]\n' + + ' wall-entry.mjs --check --file ', + ) + } + + const urlArg = args.url === true ? undefined : /** @type {string | undefined} */ (args.url) + const thumbArg = args.thumb === true ? undefined : /** @type {string | undefined} */ (args.thumb) + + const entry = deriveEntry({ + product: /** @type {string} */ (product), + version: /** @type {string} */ (version), + date: /** @type {string} */ (date), + changelogPath: /** @type {string} */ (fromChangelog), + url: urlArg, + thumb: thumbArg, + }) + + const filePath = /** @type {string} */ (args.file ?? `releases/${product}.json`) + applyEntry(entry, filePath, /** @type {string} */ (product)) +} + +main() diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts new file mode 100644 index 00000000..fc41731c --- /dev/null +++ b/tests/unit/release/wall-entry.test.ts @@ -0,0 +1,216 @@ +/** + * scripts/wall-entry.mjs — the mechanical releases-wall entry. + * + * The script's only real interface is its CLI (it has no importable + * exports by design — one door, no parallel API to drift from it), so + * these tests spawn it exactly as scripts/release.sh does: as a child + * process, against a temp copy of a wall file and a fixture CHANGELOG, + * never against the repo's real releases/*.json. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { execFileSync } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +const SCRIPT = join(process.cwd(), 'scripts/wall-entry.mjs') + +/** Run the script and capture the outcome without throwing on a non-zero exit. */ +function run(args: string[], cwd: string): { status: number; stdout: string; stderr: string } { + try { + const stdout = execFileSync('node', [SCRIPT, ...args], { cwd, encoding: 'utf8' }) + return { status: 0, stdout, stderr: '' } + } catch (err: any) { + return { status: err.status ?? 1, stdout: err.stdout ?? '', stderr: err.stderr ?? '' } + } +} + +const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' + +/** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ +function buildChangelog(entries: Array<{ version: string; date: string; bullets: string[] }>): string { + const body = entries + .map( + (e) => + `### [${e.version}](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/vX...v${e.version}) (${e.date})\n\n` + + e.bullets.map((b) => `- ${b} (abc1234)`).join('\n') + + '\n', + ) + .join('\n') + return CHANGELOG_HEADER + '\n' + body +} + +function wallFile(product: string, entries: unknown[]): string { + return JSON.stringify( + { product, entries, history: 'Earlier releases are recorded in CHANGELOG.md in this repository.' }, + null, + 2, + ) + '\n' +} + +const BASE_ENTRY = { + version: '10.4.11', + date: '2026-09-02', + headline: 'A faster open', + items: ['A faster open.'], + url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11', + thumb: null, +} + +let dir: string + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) +}) + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }) +}) + +describe('wall-entry.mjs — generate + prepend', () => { + it('derives headline from the first bullet and items from every bullet, hashes stripped', () => { + writeFileSync( + join(dir, 'CHANGELOG.md'), + buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), + ) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + expect(result.status).toBe(0) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries).toHaveLength(2) + expect(wall.entries[0]).toEqual({ + version: '10.4.12', + date: '2026-09-03', + headline: 'fix(wall): mechanize the entry', + items: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'], + url: 'https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.12', + thumb: null, + }) + // the older entry stays put, still second + expect(wall.entries[1].version).toBe('10.4.11') + }) + + it('prepends newest-first — the new entry lands at index 0 ahead of every existing one', () => { + writeFileSync( + join(dir, 'CHANGELOG.md'), + buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }]), + ) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + + run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) + }) + + it('derives no URL (null) for a product with no known public release-page pattern', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }])) + + run(['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + + const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + expect(wall.entries[0].url).toBeNull() + expect(wall.entries[0].thumb).toBeNull() + }) + + it('refuses by name when the version is already present, and leaves the file untouched', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) + const before = wallFile('open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'wall.json'), before) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/refusing.*10\.4\.11.*already present/i) + expect(readFileSync(join(dir, 'wall.json'), 'utf8')).toBe(before) // untouched + }) + + it('refuses when the CHANGELOG has no entry yet for the target version', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [])) + + const result = run( + ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) + }) + + it('refuses a cross-product write when --product does not match the target file', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + + const result = run( + ['--product', 'brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/product "open-brainy".*--product "brainy"/i) + }) +}) + +describe('wall-entry.mjs — --check', () => { + it('passes a well-formed, newest-first file with no duplicates', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/OK/) + }) + + it('catches a missing entry key', () => { + const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'], url: null } // no "thumb" + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/missing key\(s\) thumb/) + }) + + it('catches an unexpected top-level key', () => { + const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) + raw.extra = 'not allowed' + writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/unexpected key\(s\) extra/) + }) + + it('catches entries that are not newest-first', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, version: '10.4.10' }, BASE_ENTRY])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/not newest-first/) + }) + + it('catches a duplicate version even with identical entries', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/duplicate version 10\.4\.11/) + }) + + it('catches an empty items array', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, items: [] }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/"items" must be a non-empty array/) + }) + + it('catches a malformed date', () => { + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [{ ...BASE_ENTRY, date: '09/03/2026' }])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/"date" must be a YYYY-MM-DD string/) + }) +}) From adcb883e67ab82b749d37a510ab66323ae1da64e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:51:33 -0700 Subject: [PATCH 133/150] ci(release): publish the wall entry to the shared releases repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rail used to write releases/open-brainy.json (and, before that, also carried the product engine's releases/brainy.json) in this repo. It now clones (or refreshes a cached clone of) soulcraftlabs/releases on The Source, prepends the derived entry to open-brainy.json there (replacing any entry for the same version so a re-run is idempotent), and pushes main directly. Any failure — clone, shape validation, commit, or a rejected push — exits non-zero naming the cure; nothing is ever skipped. Both wall files are gone from this repo — the shared repo is the one home HQ reads. --dry-run derives and prints the entry without touching any clone or remote. Tests point --remote/--cache-dir at a throwaway local bare repo and cache dir, never the real ones. --- releases/brainy.json | 76 -------- releases/open-brainy.json | 136 ------------- scripts/release.sh | 13 +- scripts/wall-entry.mjs | 253 ++++++++++++++++++------ tests/unit/release/wall-entry.test.ts | 271 +++++++++++++++++++++----- 5 files changed, 423 insertions(+), 326 deletions(-) delete mode 100644 releases/brainy.json delete mode 100644 releases/open-brainy.json diff --git a/releases/brainy.json b/releases/brainy.json deleted file mode 100644 index 8f61c7f2..00000000 --- a/releases/brainy.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "product": "brainy", - "entries": [ - { - "version": "11.0.5", - "date": "2026-09-02", - "headline": "Graph-first finds in production, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows through a native door — correct at every page and O(neighbours), never the whole store.", - "related() with a list of verb types returns every requested kind (a fast path had silently kept only the first).", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.4", - "date": "2026-09-01", - "headline": "Closes in milliseconds, index rebuilds without the disk-sync storm", - "items": [ - "close() no longer pays deferred compaction or waits out an in-flight rebuild — measured 8 ms against the 4-minute closes it replaces; deferred work resumes at the next open, in the background.", - "The metadata index's rebuild syncs to disk per shard instead of per row, and the durability point moved to the publish step — the same guarantee, a fraction of the disk traffic.", - "A new native filter door evaluates queries over exactly the candidate rows a graph walk found, never the whole store." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.3", - "date": "2026-09-01", - "headline": "The embedding upgrade ceremony runs on every brain", - "items": [ - "A brain opened through the standard plugin now carries its embedding-model identity, so the full-precision upgrade ceremony can run on it.", - "A one-fix release; nothing else changed." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.2", - "date": "2026-08-31", - "headline": "One embedding quality everywhere, 3–4× faster imports", - "items": [ - "Every runtime embeds with the same full-precision model — search quality no longer depends on where you run.", - "Bulk embedding measured 3.1–4.2× faster, and an online re-embed ceremony upgrades existing stores without downtime.", - "The engine's change feed is documented, with the SSE/WebSocket fan-out pattern for realtime surfaces." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.1", - "date": "2026-08-31", - "headline": "Deletes inside transactions are safe", - "items": [ - "Deleting relations inside a transact() no longer corrupts index bookkeeping.", - "A store that deletes its last relation keeps serving instead of refusing." - ], - "url": null, - "thumb": null - }, - { - "version": "11.0.0", - "date": "2026-08-28", - "headline": "One install, one engine — Brainy", - "items": [ - "The former two-package pair is one package: the native engine under the familiar API. One import is the whole install.", - "A missing native build refuses loudly with its cures named; nothing falls back silently.", - "Stores open in place — no migration." - ], - "url": null, - "thumb": null - } - ], - "history": "The version line continues from the 4.3.x native-engine releases; their record lives in the product repository's CHANGELOG.md." -} diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 9f1cd239..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.11", - "date": "2026-09-02", - "headline": "Hybrid finds filter before they hydrate, one owner per shutdown, and a faster open", - "items": [ - "Hybrid finds (query/vector combined with a filter, including connected and fusion finds) now filter first and hydrate only the page — one batchGet of exactly the requested rows, instead of hydrating everything the search side found. Fixes a bug where any page after the first came back empty.", - "A brain now has exactly one shutdown owner — a host and its engine no longer race to close the same store, and a follow-up flush requested during a running flush is handed off cleanly instead of ever risking a stall.", - "find({ path }) and other path-scoped VFS searches now serve a real range over the indexed path (O(log n)) instead of refusing the query outright — both scoped and recursive:false searches were silently broken before this.", - "Open no longer rescans a brain's whole fact log on every open — sealed segments the manifest already accounts for are skipped, collapsing a multi-second open term to near-zero on large brains.", - "commitTransaction() now refuses by name if single-ops are still pending, and a read-only open no longer writes clean-shutdown evidence it didn't earn — two correctness invariants that were previously assumed, not enforced." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11", - "thumb": null - }, - { - "version": "10.4.10", - "date": "2026-09-02", - "headline": "A planner door for indexes, batched containment repair, and a fixed near()", - "items": [ - "An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.", - "repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.", - "find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10", - "thumb": null - }, - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} diff --git a/scripts/release.sh b/scripts/release.sh index 07d225ce..142fa06f 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -176,16 +176,21 @@ fi echo -e "${GREEN}✅ CHANGELOG updated${NC}\n" # Step 6b: Update the releases wall entry — mechanical, derived from the -# CHANGELOG entry just composed. The fleet's HQ page reads releases/open-brainy.json -# directly; this used to be hand-written after every release (David: never -# again — make it a step of the rail). +# CHANGELOG entry just composed. The fleet's HQ page reads open-brainy.json +# from the one shared releases repo, soulcraftlabs/releases on The Source — +# this used to be hand-written after every release (David: never again — +# make it a step of the rail, landed in the one shared home; this repo no +# longer hosts its own copy). This step clones/fetches that repo into a +# local cache, prepends the entry, and pushes it directly — a real +# cross-repo push, refusing loudly (never skipping) on any +# clone/validation/commit/push failure. echo -e "${BLUE}5️⃣▸ Updating the releases wall...${NC}" node scripts/wall-entry.mjs --product open-brainy --version "${NEW_VERSION}" --date "${RELEASE_DATE}" --from-changelog CHANGELOG.md echo -e "${GREEN}✅ Releases wall updated${NC}\n" # Step 7: Create release commit echo -e "${BLUE}6️⃣ Creating release commit...${NC}" -git add package.json package-lock.json CHANGELOG.md releases/open-brainy.json +git add package.json package-lock.json CHANGELOG.md git commit -m "chore(release): ${NEW_VERSION}" echo -e "${GREEN}✅ Release commit created${NC}\n" diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs index 998431da..043341eb 100644 --- a/scripts/wall-entry.mjs +++ b/scripts/wall-entry.mjs @@ -2,40 +2,76 @@ /** * @module scripts/wall-entry * @description The releases-wall entry, made mechanical. The fleet's HQ page - * reads one public JSON per product (releases/.json — shape - * {product, entries:[{version, date, headline, items, url, thumb}], history}). - * Those entries were hand-written after every release; this script is the - * one door that composes one, so it never has to be typed by hand again. + * reads one public JSON per product from the ONE releases repo on The Source + * (soulcraftlabs/releases, files .json at its root — shape + * {product, entries:[{version, date, headline, items, url, thumb?}]}), at + * https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/.json. + * Those entries were hand-written after every release, then briefly written + * into this repo's own releases/.json; this script is the one door + * that composes an entry and lands it in the shared repo, so it is never + * hand-written and never forked across repos again. * * Two modes: * - * 1. Generate + write in place (default): + * 1. Generate + publish (default): * node wall-entry.mjs --product

--version --date \ - * --from-changelog [--file releases/

.json] + * --from-changelog * Derives an entry from the CHANGELOG.md entry for (headline = the * entry's first bullet, items = every bullet, trimmed of its trailing - * commit hash), prepends it to --file (default releases/.json, - * newest first), refusing by name if is already present, and - * validates the whole file's shape + ordering before and after writing. - * Both engines run this identically, each against its own repo's - * releases/.json — the wall file always lives beside the - * CHANGELOG it is derived from, never in another repo. + * commit hash), then: + * - clones (or, if a cached clone already exists, fetches and resets) + * the releases repo into a local cache directory, + * - prepends the entry to /

.json, newest first — replacing + * any existing entry for the same version so a re-run is idempotent, + * - validates the file's shape before and after, + * - commits the change as "chore(wall):

" and pushes main. + * A failure at any step (clone, validation, commit, push, a + * non-fast-forward remote) exits non-zero naming the cure. Nothing is + * ever skipped — the wall either lands correctly or the release fails. * - * 2. Validate only (--check): - * node wall-entry.mjs --check --file - * Validates the file's exact key set (top-level and per-entry), field - * types, and strict-descending semver ordering with no duplicates. - * Read-only; never writes. Exit 0 = clean, exit 1 = named violations - * printed to stderr. + * 2. Dry run: + * node wall-entry.mjs --dry-run --product

--version \ + * --date --from-changelog + * Derives the entry exactly as above and prints it, along with the file + * it would be written to, but touches no clone and no remote — usable + * from a fresh checkout with no cache and no network. * - * No dependencies — CHANGELOG parsing, semver comparison, and JSON shape - * checking are all hand-rolled below. + * 3. Validate only (--check): + * node wall-entry.mjs --check --file + * Validates an arbitrary wall file's exact key set (top-level and + * per-entry), field types, and strict-descending semver ordering with + * no duplicates. Read-only; never writes. Exit 0 = clean, exit 1 = + * named violations printed to stderr. + * + * The remote and the local cache directory are each overridable + * (--remote / --cache-dir, or WALL_ENTRY_RELEASES_REMOTE / + * WALL_ENTRY_RELEASES_CACHE_DIR) so tests can point at a throwaway local + * bare repo and a throwaway cache directory — never the real remote or the + * real developer cache. + * + * No dependencies beyond the system `git` binary — CHANGELOG parsing, + * semver comparison, and JSON shape checking are all hand-rolled below. */ -import { readFileSync, writeFileSync, existsSync } from 'node:fs' +import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs' +import { execFileSync } from 'node:child_process' +import { homedir } from 'node:os' +import { dirname, join } from 'node:path' -const ENTRY_KEYS = ['version', 'date', 'headline', 'items', 'url', 'thumb'] -const FILE_KEYS = ['product', 'entries', 'history'] +const DEFAULT_REMOTE = 'git@source.soulcraft.com:soulcraftlabs/releases.git' + +/** @returns {string} */ +function defaultCacheDir() { + const base = process.env.XDG_CACHE_HOME || join(homedir(), '.cache') + return join(base, 'soulcraft-releases') +} + +// Required on every entry; "thumb" is optional (may be absent, or present as +// string | null) — matching the HQ contract's {..., thumb?}. +const ENTRY_REQUIRED_KEYS = ['version', 'date', 'headline', 'items', 'url'] +const ENTRY_OPTIONAL_KEYS = ['thumb'] +const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS] +const FILE_KEYS = ['product', 'entries'] // The public release-page URL pattern, by product — only products with a // PUBLIC forge repo get a derived link. A product without an entry here @@ -110,10 +146,11 @@ function compareSemver(a, b) { } /** - * Validate a wall file's full shape: top-level keys, per-entry keys and - * field types, and strict-descending semver ordering with no duplicates. - * Collects every violation instead of failing on the first, so --check - * reports the whole picture in one pass. + * Validate a wall file's full shape: top-level keys ("product", "entries" — + * no more, no less), per-entry keys and field types ("thumb" optional), and + * strict-descending semver ordering with no duplicates. Collects every + * violation instead of failing on the first, so a caller reports the whole + * picture in one pass. * @param {unknown} data * @returns {string[]} Violation messages; empty means the file is clean. */ @@ -135,9 +172,6 @@ function validateShape(data) { if (typeof obj.product !== 'string' || obj.product.trim() === '') { errors.push('top level: "product" must be a non-empty string') } - if (typeof obj.history !== 'string' || obj.history.trim() === '') { - errors.push('top level: "history" must be a non-empty string') - } if (!Array.isArray(obj.entries)) { errors.push('top level: "entries" must be an array') return errors // nothing further to check without an array @@ -152,8 +186,8 @@ function validateShape(data) { } const entry = /** @type {Record} */ (rawEntry) const keys = Object.keys(entry) - const missing = ENTRY_KEYS.filter((k) => !(k in entry)) - const extra = keys.filter((k) => !ENTRY_KEYS.includes(k)) + const missing = ENTRY_REQUIRED_KEYS.filter((k) => !(k in entry)) + const extra = keys.filter((k) => !ENTRY_ALLOWED_KEYS.includes(k)) if (missing.length) errors.push(`${label}: missing key(s) ${missing.join(', ')}`) if (extra.length) errors.push(`${label}: unexpected key(s) ${extra.join(', ')}`) @@ -172,8 +206,8 @@ function validateShape(data) { if (!(entry.url === null || typeof entry.url === 'string')) { errors.push(`${label}: "url" must be a string or null`) } - if (!(entry.thumb === null || typeof entry.thumb === 'string')) { - errors.push(`${label}: "thumb" must be a string or null`) + if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) { + errors.push(`${label}: "thumb" must be a string or null when present`) } }) @@ -266,43 +300,110 @@ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { * @returns {Record} */ function loadWallFile(filePath) { - if (!existsSync(filePath)) fail(`--file "${filePath}" does not exist`) + if (!existsSync(filePath)) fail(`"${filePath}" does not exist`) /** @type {unknown} */ let data try { data = JSON.parse(readFileSync(filePath, 'utf8')) } catch (err) { - fail(`--file "${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) + fail(`"${filePath}" is not valid JSON: ${/** @type {Error} */ (err).message}`) } const errors = validateShape(data) if (errors.length) { - fail(`--file "${filePath}" fails shape validation before any write —\n ${errors.join('\n ')}`) + fail(`"${filePath}" fails shape validation —\n ${errors.join('\n ')}`) } return /** @type {Record} */ (data) } /** - * Prepend `entry` to the wall file at `filePath`, refusing by name if the - * version is already present, validating before and after, and writing the - * file back with the repo's exact formatting (2-space JSON, trailing newline). - * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry - * @param {string} filePath - * @param {string | undefined} expectedProduct + * Run a git command, throwing an Error whose message is git's own stderr + * (trimmed) on failure — every caller wraps this to name the cure. + * @param {string[]} args + * @param {string} cwd + * @returns {string} stdout, trimmed. */ -function applyEntry(entry, filePath, expectedProduct) { - const wall = loadWallFile(filePath) +function git(args, cwd) { + try { + return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim() + } catch (err) { + const stderr = /** @type {any} */ (err).stderr + const message = (typeof stderr === 'string' && stderr.trim()) || /** @type {Error} */ (err).message + throw new Error(message) + } +} - if (expectedProduct && wall.product !== expectedProduct) { +/** + * Ensure a clean, up-to-date local clone of the releases repo at + * `cacheDir`, checked out on `main` — cloning fresh if `cacheDir` has no + * `.git`, otherwise fetching and hard-resetting onto `origin/main` (so a + * stray local commit or edit left by a previous failed run can never leak + * into the next one). + * @param {string} remote + * @param {string} cacheDir + */ +function ensureReleasesClone(remote, cacheDir) { + if (existsSync(join(cacheDir, '.git'))) { + try { + git(['remote', 'set-url', 'origin', remote], cacheDir) + git(['fetch', '--prune', 'origin'], cacheDir) + git(['checkout', 'main'], cacheDir) + git(['reset', '--hard', 'origin/main'], cacheDir) + git(['clean', '-fd'], cacheDir) + } catch (err) { + fail( + `cannot refresh the cached releases checkout at "${cacheDir}" from "${remote}" — ${/** @type {Error} */ (err).message}\n` + + ` cure: delete "${cacheDir}" and re-run so it re-clones from scratch, or confirm SSH access with "ssh -T git@source.soulcraft.com"`, + ) + } + return + } + + mkdirSync(dirname(cacheDir), { recursive: true }) + try { + git(['clone', remote, cacheDir], dirname(cacheDir)) + } catch (err) { fail( - `--file "${filePath}" has product "${wall.product}", but --product "${expectedProduct}" was given — refusing a cross-product write`, + `cannot clone "${remote}" — ${/** @type {Error} */ (err).message}\n` + + ` cure: confirm SSH access with "ssh -T git@source.soulcraft.com" and that the soulcraftlabs/releases repo exists yet`, ) } + try { + git(['checkout', 'main'], cacheDir) + } catch (err) { + fail( + `cloned "${remote}" into "${cacheDir}" but could not check out "main" — ${/** @type {Error} */ (err).message}\n` + + ` cure: confirm the releases repo's default branch is named "main"`, + ) + } +} - if (wall.entries.some((e) => e.version === entry.version)) { - fail(`refusing — version ${entry.version} is already present in "${filePath}"`) +/** + * Prepend `entry` to the wall at `/.json`, replacing any + * existing entry for the same version (idempotent re-runs), validating + * before and after, committing, and pushing — or refusing loudly, naming + * the cure, at whichever step fails. + * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {string} product + * @param {string} remote + * @param {string} cacheDir + */ +function publishEntry(entry, product, remote, cacheDir) { + ensureReleasesClone(remote, cacheDir) + + const filePath = join(cacheDir, `${product}.json`) + if (!existsSync(filePath)) { + fail( + `"${filePath}" does not exist in the releases repo — cure: seed "${product}.json" at the repo root first (it must exist before any release rail can prepend to it)`, + ) + } + const wall = loadWallFile(filePath) + + if (wall.product !== product) { + fail(`"${filePath}" has product "${wall.product}", but --product "${product}" was given — refusing a cross-product write`) } - wall.entries = [entry, ...wall.entries] + const replacing = wall.entries.some((e) => e.version === entry.version) + wall.entries = [entry, ...wall.entries.filter((e) => e.version !== entry.version)] const postErrors = validateShape(wall) if (postErrors.length) { @@ -310,22 +411,48 @@ function applyEntry(entry, filePath, expectedProduct) { } writeFileSync(filePath, JSON.stringify(wall, null, 2) + '\n', 'utf8') - console.log(`wall-entry: wrote v${entry.version} to "${filePath}" (${wall.entries.length} entries, newest first)`) + + const status = git(['status', '--porcelain', '--', `${product}.json`], cacheDir) + if (status === '') { + console.log(`wall-entry: "${product}.json" already carries an identical entry for ${entry.version} — nothing to commit or push`) + return + } + + try { + git(['add', `${product}.json`], cacheDir) + git(['commit', '-m', `chore(wall): ${product} ${entry.version}`], cacheDir) + } catch (err) { + fail(`cannot commit the wall entry in "${cacheDir}" — ${/** @type {Error} */ (err).message}\n cure: inspect "${cacheDir}" by hand and re-run once its git state is clean`) + } + + try { + git(['push', 'origin', 'main'], cacheDir) + } catch (err) { + fail( + `push to "${remote}" failed (likely a non-fast-forward — another release landed on main first) — ${/** @type {Error} */ (err).message}\n` + + ` cure: re-run this release step; it re-fetches and resets onto the latest origin/main before retrying`, + ) + } + + const sha = git(['rev-parse', 'HEAD'], cacheDir) + console.log( + `wall-entry: ${replacing ? 'replaced' : 'wrote'} v${entry.version} in "${product}.json" (${wall.entries.length} entries, newest first) — pushed ${sha} to ${remote} main`, + ) } function main() { const args = parseArgs(process.argv.slice(2)) if (args.check) { - const filePath = /** @type {string | undefined} */ (args.file) ?? - (typeof args.product === 'string' ? `releases/${args.product}.json` : undefined) - if (!filePath) fail('--check needs --file (or --product to default to releases/.json)') + const filePath = /** @type {string | undefined} */ (args.file) + if (!filePath) fail('--check needs --file ') const wall = loadWallFile(/** @type {string} */ (filePath)) console.log(`wall-entry --check: "${filePath}" OK — product "${wall.product}", ${wall.entries.length} entries, newest-first, no duplicates`) process.exit(0) } - // Generate mode (default): --product, --version, --date, --from-changelog required. + // Generate mode (default, also covers --dry-run): --product, --version, + // --date, --from-changelog required. const product = /** @type {string | undefined} */ (args.product) const version = /** @type {string | undefined} */ (args.version) const date = /** @type {string | undefined} */ (args.date) @@ -340,8 +467,8 @@ function main() { fail( `missing required flag(s): ${missing.join(', ')}\n` + 'Usage:\n' + - ' wall-entry.mjs --product

--version --date --from-changelog [--file releases/

.json]\n' + - ' wall-entry.mjs --check --file ', + ' wall-entry.mjs --product

--version --date --from-changelog [--dry-run]\n' + + ' wall-entry.mjs --check --file ', ) } @@ -357,8 +484,16 @@ function main() { thumb: thumbArg, }) - const filePath = /** @type {string} */ (args.file ?? `releases/${product}.json`) - applyEntry(entry, filePath, /** @type {string} */ (product)) + const remote = /** @type {string} */ (args.remote ?? process.env.WALL_ENTRY_RELEASES_REMOTE ?? DEFAULT_REMOTE) + const cacheDir = /** @type {string} */ (args['cache-dir'] ?? process.env.WALL_ENTRY_RELEASES_CACHE_DIR ?? defaultCacheDir()) + + if (args['dry-run']) { + console.log(`wall-entry --dry-run: would write to "${join(cacheDir, `${product}.json`)}" in ${remote} (main), pushed as "chore(wall): ${product} ${version}"`) + console.log(JSON.stringify(entry, null, 2)) + process.exit(0) + } + + publishEntry(entry, /** @type {string} */ (product), remote, cacheDir) } main() diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts index fc41731c..7f96da25 100644 --- a/tests/unit/release/wall-entry.test.ts +++ b/tests/unit/release/wall-entry.test.ts @@ -4,12 +4,15 @@ * The script's only real interface is its CLI (it has no importable * exports by design — one door, no parallel API to drift from it), so * these tests spawn it exactly as scripts/release.sh does: as a child - * process, against a temp copy of a wall file and a fixture CHANGELOG, - * never against the repo's real releases/*.json. + * process, against a fixture CHANGELOG and a throwaway local bare repo + * standing in for git@source.soulcraft.com:soulcraftlabs/releases.git + * (--remote) plus a throwaway cache directory (--cache-dir) standing in + * for ~/.cache/soulcraft-releases — never the real remote, never the + * real developer cache. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { execFileSync } from 'node:child_process' -import { mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs' +import { mkdtempSync, rmSync, writeFileSync, readFileSync, chmodSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -25,6 +28,10 @@ function run(args: string[], cwd: string): { status: number; stdout: string; std } } +function git(args: string[], cwd: string): string { + return execFileSync('git', ['-C', cwd, ...args], { encoding: 'utf8' }).trim() +} + const CHANGELOG_HEADER = '# Changelog\n\nAll notable changes, in this fixture.\n' /** Build a CHANGELOG.md with one entry per [version, bullets[]] pair, newest first. */ @@ -41,11 +48,7 @@ function buildChangelog(entries: Array<{ version: string; date: string; bullets: } function wallFile(product: string, entries: unknown[]): string { - return JSON.stringify( - { product, entries, history: 'Earlier releases are recorded in CHANGELOG.md in this repository.' }, - null, - 2, - ) + '\n' + return JSON.stringify({ product, entries }, null, 2) + '\n' } const BASE_ENTRY = { @@ -57,31 +60,76 @@ const BASE_ENTRY = { thumb: null, } +/** A throwaway bare repo standing in for the real soulcraftlabs/releases remote. */ +function initBareRemote(): string { + const remoteDir = mkdtempSync(join(tmpdir(), 'wall-remote-')) + execFileSync('git', ['init', '--bare', '-b', 'main', remoteDir]) + return remoteDir +} + +/** Seed the bare remote with an initial .json, via a throwaway clone. */ +function seedRemote(remoteDir: string, product: string, entries: unknown[]): void { + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + writeFileSync(join(seedDir, `${product}.json`), wallFile(product, entries)) + git(['add', `${product}.json`], seedDir) + git(['commit', '-m', 'seed'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) +} + +/** Read .json back out of the bare remote's main tip, via a throwaway clone. */ +function readRemote(remoteDir: string, product: string): any { + const readDir = mkdtempSync(join(tmpdir(), 'wall-read-')) + execFileSync('git', ['clone', remoteDir, readDir], { stdio: 'ignore' }) + const data = JSON.parse(readFileSync(join(readDir, `${product}.json`), 'utf8')) + rmSync(readDir, { recursive: true, force: true }) + return data +} + +/** Reject every push — stands in for any push failure (including a genuine + * non-fast-forward raced by a concurrent release rail), which this script + * treats identically: refuse loudly, name the cure, touch nothing further. */ +function makeRemoteRejectPushes(remoteDir: string): void { + const hookPath = join(remoteDir, 'hooks', 'pre-receive') + writeFileSync(hookPath, '#!/bin/sh\necho "remote: simulated push rejection" >&2\nexit 1\n') + chmodSync(hookPath, 0o755) +} + let dir: string +let remoteDir: string +let cacheDir: string beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'wall-entry-test-')) + remoteDir = initBareRemote() + cacheDir = join(mkdtempSync(join(tmpdir(), 'wall-cache-')), 'soulcraft-releases') }) afterEach(() => { rmSync(dir, { recursive: true, force: true }) + rmSync(remoteDir, { recursive: true, force: true }) + rmSync(cacheDir, { recursive: true, force: true }) }) -describe('wall-entry.mjs — generate + prepend', () => { - it('derives headline from the first bullet and items from every bullet, hashes stripped', () => { +describe('wall-entry.mjs — generate + publish', () => { + it('derives headline from the first bullet and items from every bullet, hashes stripped, and pushes it to the remote', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) writeFileSync( join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix(wall): mechanize the entry', 'test(wall): pin the shape'] }]), ) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) const result = run( - ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(0) + expect(result.stdout).toMatch(/wrote v10\.4\.12.*pushed/i) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'open-brainy') expect(wall.entries).toHaveLength(2) expect(wall.entries[0]).toEqual({ version: '10.4.12', @@ -96,68 +144,182 @@ describe('wall-entry.mjs — generate + prepend', () => { }) it('prepends newest-first — the new entry lands at index 0 ahead of every existing one', () => { - writeFileSync( - join(dir, 'CHANGELOG.md'), - buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }]), - ) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }])) + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY, { ...BASE_ENTRY, version: '10.4.10' }]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.5.0', date: '2026-09-03', bullets: ['feat: ten five'] }])) - run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + run(['--product', 'open-brainy', '--version', '10.5.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'open-brainy') expect(wall.entries.map((e: any) => e.version)).toEqual(['10.5.0', '10.4.11', '10.4.10']) }) + it('replaces an entry with the same version instead of duplicating it — idempotent re-runs', () => { + seedRemote(remoteDir, 'open-brainy', [ + { ...BASE_ENTRY, headline: 'stale headline, pre-fix' }, + { ...BASE_ENTRY, version: '10.4.10' }, + ]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: the corrected headline'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/replaced v10\.4\.11/i) + + const wall = readRemote(remoteDir, 'open-brainy') + expect(wall.entries).toHaveLength(2) // not 3 — replaced, not duplicated + expect(wall.entries[0].version).toBe('10.4.11') + expect(wall.entries[0].headline).toBe('fix: the corrected headline') + expect(wall.entries[1].version).toBe('10.4.10') + }) + + it('a re-run with byte-identical content commits nothing and still succeeds', () => { + // headline always equals items[0] for a derived entry, so this fixture + // (unlike BASE_ENTRY, whose headline/items intentionally diverge for the + // shape-only tests below) has to keep the two in lockstep to ever roundtrip. + const stableEntry = { ...BASE_ENTRY, headline: 'A faster open.', items: ['A faster open.'] } + seedRemote(remoteDir, 'open-brainy', [stableEntry]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['A faster open.'] }])) + const before = readRemote(remoteDir, 'open-brainy') + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/nothing to commit/i) + expect(readRemote(remoteDir, 'open-brainy')).toEqual(before) + }) + it('derives no URL (null) for a product with no known public release-page pattern', () => { + seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }]) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }])) - run(['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], dir) + const result = run( + ['--product', 'brainy', '--version', '11.0.6', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + expect(result.status).toBe(0) - const wall = JSON.parse(readFileSync(join(dir, 'wall.json'), 'utf8')) + const wall = readRemote(remoteDir, 'brainy') expect(wall.entries[0].url).toBeNull() expect(wall.entries[0].thumb).toBeNull() }) - it('refuses by name when the version is already present, and leaves the file untouched', () => { + it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => { + seedRemote(remoteDir, 'open-brainy', []) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) - const before = wallFile('open-brainy', [BASE_ENTRY]) - writeFileSync(join(dir, 'wall.json'), before) + const beforeSha = git(['rev-parse', 'main'], remoteDir) const result = run( - ['--product', 'open-brainy', '--version', '10.4.11', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], - dir, - ) - - expect(result.status).toBe(1) - expect(result.stderr).toMatch(/refusing.*10\.4\.11.*already present/i) - expect(readFileSync(join(dir, 'wall.json'), 'utf8')).toBe(before) // untouched - }) - - it('refuses when the CHANGELOG has no entry yet for the target version', () => { - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [])) - - const result = run( - ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '99.0.0', '--date', '2026-09-02', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(1) expect(result.stderr).toMatch(/no CHANGELOG entry yet/i) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) }) - it('refuses a cross-product write when --product does not match the target file', () => { - writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) - writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [BASE_ENTRY])) + it('refuses by naming the cure when the remote cannot be cloned', () => { + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + const noSuchRemote = join(tmpdir(), 'wall-remote-does-not-exist-' + Date.now()) const result = run( - ['--product', 'brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--file', 'wall.json'], + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', noSuchRemote, '--cache-dir', cacheDir], dir, ) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/product "open-brainy".*--product "brainy"/i) + expect(result.stderr).toMatch(/cannot clone/i) + expect(result.stderr).toMatch(/cure:/i) + }) + + it('refuses by naming the cure, and touches no remote, when the fetched wall fails shape validation', () => { + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-broken-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + writeFileSync( + join(seedDir, 'open-brainy.json'), + JSON.stringify({ product: 'open-brainy', entries: [{ version: '10.4.11', date: '2026-09-02', items: ['x'], url: null }] }, null, 2), + ) + git(['add', 'open-brainy.json'], seedDir) + git(['commit', '-m', 'seed broken'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) + const beforeSha = git(['rev-parse', 'main'], remoteDir) + + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/fails shape validation/i) + expect(result.stderr).toMatch(/missing key\(s\) headline/i) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) + }) + + it('refuses by naming the cure when the remote rejects the push (stands in for a raced non-fast-forward)', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + makeRemoteRejectPushes(remoteDir) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: whatever'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/push to .* failed/i) + expect(result.stderr).toMatch(/cure:/i) + }) + + it('refuses a cross-product write when the file\'s "product" field does not match --product', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + const seedDir = mkdtempSync(join(tmpdir(), 'wall-seed-mismatch-')) + execFileSync('git', ['clone', remoteDir, seedDir], { stdio: 'ignore' }) + git(['config', 'user.email', 'seed@example.com'], seedDir) + git(['config', 'user.name', 'Seed'], seedDir) + const corrupted = JSON.parse(readFileSync(join(seedDir, 'open-brainy.json'), 'utf8')) + corrupted.product = 'brainy' + writeFileSync(join(seedDir, 'open-brainy.json'), JSON.stringify(corrupted, null, 2) + '\n') + git(['add', 'open-brainy.json'], seedDir) + git(['commit', '-m', 'corrupt product field'], seedDir) + git(['push', 'origin', 'main'], seedDir) + rmSync(seedDir, { recursive: true, force: true }) + + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['fix: wrong repo'] }])) + + const result = run( + ['--product', 'open-brainy', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(1) + expect(result.stderr).toMatch(/product "brainy".*--product "open-brainy"/i) + }) +}) + +describe('wall-entry.mjs — --dry-run', () => { + it('prints the entry and the target path, and touches neither the cache dir nor the remote', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.12', date: '2026-09-03', bullets: ['fix: a dry run'] }])) + const beforeSha = git(['rev-parse', 'main'], remoteDir) + + const result = run( + ['--dry-run', '--product', 'open-brainy', '--version', '10.4.12', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], + dir, + ) + + expect(result.status).toBe(0) + expect(result.stdout).toMatch(/would write to/i) + expect(result.stdout).toMatch(/"version": "10\.4\.12"/) + expect(git(['rev-parse', 'main'], remoteDir)).toBe(beforeSha) }) }) @@ -169,21 +331,28 @@ describe('wall-entry.mjs — --check', () => { expect(result.stdout).toMatch(/OK/) }) + it('passes a file where "thumb" is entirely absent (optional per the HQ contract)', () => { + const { thumb, ...noThumb } = BASE_ENTRY as any + writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [noThumb])) + const result = run(['--check', '--file', 'wall.json'], dir) + expect(result.status).toBe(0) + }) + it('catches a missing entry key', () => { - const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'], url: null } // no "thumb" + const broken = { version: '1.0.0', date: '2026-09-03', headline: 'h', items: ['i'] } // no "url" writeFileSync(join(dir, 'wall.json'), wallFile('open-brainy', [broken])) const result = run(['--check', '--file', 'wall.json'], dir) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/missing key\(s\) thumb/) + expect(result.stderr).toMatch(/missing key\(s\) url/) }) - it('catches an unexpected top-level key', () => { + it('catches an unexpected top-level key (e.g. the retired "history" field)', () => { const raw = JSON.parse(wallFile('open-brainy', [BASE_ENTRY])) - raw.extra = 'not allowed' + raw.history = 'retired field' writeFileSync(join(dir, 'wall.json'), JSON.stringify(raw)) const result = run(['--check', '--file', 'wall.json'], dir) expect(result.status).toBe(1) - expect(result.stderr).toMatch(/unexpected key\(s\) extra/) + expect(result.stderr).toMatch(/unexpected key\(s\) history/) }) it('catches entries that are not newest-first', () => { From aa457d715937142607245f93437f987ba00248f0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:52:05 -0700 Subject: [PATCH 134/150] chore(releases): both walls leave the reference repo, RELEASES.md points home MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit releases/open-brainy.json follows brainy.json out — the shared repo (soulcraftlabs/releases on The Source) is now the one home for both products' release notes; this repo hosts neither. The releases/ directory is gone. RELEASES.md gains a pointer, under the heading, to the two raw URLs HQ's /hq/releases door reads (this file stays as the human-readable quick reference; those files are the source of truth). --- RELEASES.md | 7 ++ releases/open-brainy.json | 136 -------------------------------------- 2 files changed, 7 insertions(+), 136 deletions(-) delete mode 100644 releases/open-brainy.json diff --git a/RELEASES.md b/RELEASES.md index e8833b80..c875cb26 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,5 +1,12 @@ # @soulcraft/brainy — Release Notes for Consumers +Machine-readable release notes are published at +https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/open-brainy.json +(this engine) and +https://source.soulcraft.com/soulcraftlabs/releases/raw/branch/main/brainy.json +(the product engine) — read by HQ's `/hq/releases` door, and the source of +truth ahead of this file. + This file is the **quick reference for downstream sessions** tracking Brainy changes. Full auto-generated changelog: `CHANGELOG.md` · Releases: https://source.soulcraft.com/soulcraftlabs/open-brainy/releases diff --git a/releases/open-brainy.json b/releases/open-brainy.json deleted file mode 100644 index 9f1cd239..00000000 --- a/releases/open-brainy.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "product": "open-brainy", - "entries": [ - { - "version": "10.4.11", - "date": "2026-09-02", - "headline": "Hybrid finds filter before they hydrate, one owner per shutdown, and a faster open", - "items": [ - "Hybrid finds (query/vector combined with a filter, including connected and fusion finds) now filter first and hydrate only the page — one batchGet of exactly the requested rows, instead of hydrating everything the search side found. Fixes a bug where any page after the first came back empty.", - "A brain now has exactly one shutdown owner — a host and its engine no longer race to close the same store, and a follow-up flush requested during a running flush is handed off cleanly instead of ever risking a stall.", - "find({ path }) and other path-scoped VFS searches now serve a real range over the indexed path (O(log n)) instead of refusing the query outright — both scoped and recursive:false searches were silently broken before this.", - "Open no longer rescans a brain's whole fact log on every open — sealed segments the manifest already accounts for are skipped, collapsing a multi-second open term to near-zero on large brains.", - "commitTransaction() now refuses by name if single-ops are still pending, and a read-only open no longer writes clean-shutdown evidence it didn't earn — two correctness invariants that were previously assumed, not enforced." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.11", - "thumb": null - }, - { - "version": "10.4.10", - "date": "2026-09-02", - "headline": "A planner door for indexes, batched containment repair, and a fixed near()", - "items": [ - "An optional planFindPage door lets an index plan a find() and answer it in one call, instead of the engine assembling the plan itself.", - "repairContainment's reconcile pass now walks paged edges once instead of issuing one graph call per file.", - "find({ near }) now searches around the anchor's own vector and refuses by name when none is available, instead of silently querying with no vector at all." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.10", - "thumb": null - }, - { - "version": "10.4.9", - "date": "2026-09-02", - "headline": "Graph-first finds, honest verb arrays, and opens that stop rescanning history", - "items": [ - "find({ connected, where }) now walks the neighbours first and filters only those rows — correct at every page, and O(neighbours) instead of O(store).", - "related() with a list of verb types (or sources, or targets) returns every requested kind — four fast paths silently kept only the first.", - "Deferred-embedding recovery resumes from a low-water mark instead of rescanning the whole generation log at every open — measured at two minutes on a large brain, now milliseconds." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.9", - "thumb": null - }, - { - "version": "10.4.7", - "date": "2026-09-01", - "headline": "Count ledgers can no longer race themselves", - "items": [ - "Concurrent count flushes coalesce into one writer with a trailing pass — parallel flushes can no longer corrupt a store's count ledger.", - "Atomic writes carry a per-process sequence, so two processes' temp files can never collide." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.7", - "thumb": null - }, - { - "version": "10.4.6", - "date": "2026-08-31", - "headline": "Transactions cross the index seam safely", - "items": [ - "Deleting relations inside a transact() no longer fails against the metadata index — operations take a JSON-safe view at the moment they execute.", - "Fixes a class of transaction failures on stores with integer-mapped relation endpoints." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.6", - "thumb": null - }, - { - "version": "10.4.5", - "date": "2026-08-31", - "headline": "Recovery tells the truth, docs live at home", - "items": [ - "A torn generation-log tail is a terminal verdict with a named cure — never an endless wait at open.", - "A sealed segment declares only the generations it actually holds.", - "The engine's documentation now publishes from its own repository." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.5", - "thumb": null - }, - { - "version": "10.4.4", - "date": "2026-08-28", - "headline": "Faster opens, quieter idle", - "items": [ - "Opening a store discovers generations from directory names instead of walking the log, and answers \"any entities?\" with one directory read.", - "The flush-request watch is event-driven; idle stores stop paying a polling heartbeat.", - "A slow open now names the exact step it is in, so operators see what is being paid and why." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.4", - "thumb": null - }, - { - "version": "10.4.3", - "date": "2026-08-27", - "headline": "Open Brainy, under its own name", - "items": [ - "The same engine as 10.4.2, now published as @soulcraftlabs/brainy — the MIT reference engine, on The Source.", - "No code changes; your imports change once and everything else stays put." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.3", - "thumb": null - }, - { - "version": "10.4.2", - "date": "2026-08-27", - "headline": "Vectors that lie are refused, counts that drift are caught", - "items": [ - "A zero-norm vector is not a vector: the index refuses them, rebuilds skip them, and a sanctioned unvector door removes them cleanly.", - "The canonical count ledger derives from identity records and marks legacy-derived ledgers suspect at load.", - "Plugin activation failures keep their original error as cause, so the real frame reaches your logs." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.2", - "thumb": null - }, - { - "version": "10.4.1", - "date": "2026-08-26", - "headline": "Writes that change nothing cost nothing", - "items": [ - "The read gate is per index family, and a write carrying unchanged data never re-embeds.", - "The vectored-row count joins the ledger, so vector coverage is a number you can read, not a guess." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.1", - "thumb": null - }, - { - "version": "10.4.0", - "date": "2026-08-26", - "headline": "Repair routing, the vector ledger, and honest empties", - "items": [ - "Repairs route to the index that owns the damage, and the open gate closes the vector leg until coverage is proven.", - "An empty string is real data, not a missing field.", - "The metadata crossing never carries raw integer relation endpoints — a whole class of serialization faults closed." - ], - "url": "https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v10.4.0", - "thumb": null - } - ], - "history": "Earlier releases are recorded in CHANGELOG.md in this repository." -} From 97b5ea2d5ddb739f1d1d0ff4e31664b5ef551df4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 14:59:49 -0700 Subject: [PATCH 135/150] =?UTF-8?q?fix(wall):=20every=20entry=20carries=20?= =?UTF-8?q?an=20https=20permalink=20=E2=80=94=20the=20product=20engine=20l?= =?UTF-8?q?inks=20its=20public=20package=20page;=20null=20refused,=20an=20?= =?UTF-8?q?unknown=20product=20refuses=20by=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/wall-entry.mjs | 27 ++++++++++++++++----------- tests/unit/release/wall-entry.test.ts | 16 +++++++++++++--- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/scripts/wall-entry.mjs b/scripts/wall-entry.mjs index 043341eb..d4ec7ba5 100644 --- a/scripts/wall-entry.mjs +++ b/scripts/wall-entry.mjs @@ -73,13 +73,14 @@ const ENTRY_OPTIONAL_KEYS = ['thumb'] const ENTRY_ALLOWED_KEYS = [...ENTRY_REQUIRED_KEYS, ...ENTRY_OPTIONAL_KEYS] const FILE_KEYS = ['product', 'entries'] -// The public release-page URL pattern, by product — only products with a -// PUBLIC forge repo get a derived link. A product without an entry here -// (e.g. "brainy", whose repo is private) gets url: null, matching every -// entry the fleet has shipped for it so far — a private link would 404 for -// anyone reading the public HQ page. +// The public permalink pattern, by product. Every entry MUST carry an https +// permalink: HQ's parser rejects a wall whose entries carry url: null (the +// whole feed became unreadable on 2026-09-02). A product whose forge repo is +// private links its PUBLIC package page on The Source instead of a release +// page that would 404 for HQ's readers. const RELEASE_URL_PATTERNS = { 'open-brainy': (version) => `https://source.soulcraft.com/soulcraftlabs/open-brainy/releases/tag/v${version}`, + 'brainy': (version) => `https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/${version}`, } /** @@ -203,8 +204,8 @@ function validateShape(data) { if (!Array.isArray(entry.items) || entry.items.length === 0 || entry.items.some((it) => typeof it !== 'string' || it.trim() === '')) { errors.push(`${label}: "items" must be a non-empty array of non-empty strings`) } - if (!(entry.url === null || typeof entry.url === 'string')) { - errors.push(`${label}: "url" must be a string or null`) + if (typeof entry.url !== 'string' || !/^https:\/\/\S+$/.test(entry.url)) { + errors.push(`${label}: "url" must be an https permalink — never null; HQ's parser rejects the whole feed`) } if ('thumb' in entry && !(entry.thumb === null || typeof entry.thumb === 'string')) { errors.push(`${label}: "thumb" must be a string or null when present`) @@ -274,8 +275,8 @@ function extractChangelogBullets(changelog, version) { /** * Derive a wall entry from a CHANGELOG.md. - * @param {{product: string, version: string, date: string, changelogPath: string, url?: string | null, thumb?: string | null}} opts - * @returns {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} + * @param {{product: string, version: string, date: string, changelogPath: string, url?: string, thumb?: string | null}} opts + * @returns {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} */ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { if (!parseSemver(version)) fail(`--version "${version}" is not a semver string`) @@ -288,7 +289,11 @@ function deriveEntry({ product, version, date, changelogPath, url, thumb }) { const items = extractChangelogBullets(changelog, version) const headline = items[0] - const resolvedUrl = url !== undefined ? url : (RELEASE_URL_PATTERNS[product]?.(version) ?? null) + const pattern = RELEASE_URL_PATTERNS[product] + if (url === undefined && pattern === undefined) { + throw new Error(`wall-entry: no permalink pattern for product "${product}" — add one to RELEASE_URL_PATTERNS or pass --url; entries never carry url: null`) + } + const resolvedUrl = url !== undefined ? url : pattern(version) const resolvedThumb = thumb !== undefined ? thumb : null return { version, date, headline, items, url: resolvedUrl, thumb: resolvedThumb } @@ -382,7 +387,7 @@ function ensureReleasesClone(remote, cacheDir) { * existing entry for the same version (idempotent re-runs), validating * before and after, committing, and pushing — or refusing loudly, naming * the cure, at whichever step fails. - * @param {{version: string, date: string, headline: string, items: string[], url: string | null, thumb: string | null}} entry + * @param {{version: string, date: string, headline: string, items: string[], url: string, thumb: string | null}} entry * @param {string} product * @param {string} remote * @param {string} cacheDir diff --git a/tests/unit/release/wall-entry.test.ts b/tests/unit/release/wall-entry.test.ts index 7f96da25..8bf9d357 100644 --- a/tests/unit/release/wall-entry.test.ts +++ b/tests/unit/release/wall-entry.test.ts @@ -192,8 +192,8 @@ describe('wall-entry.mjs — generate + publish', () => { expect(readRemote(remoteDir, 'open-brainy')).toEqual(before) }) - it('derives no URL (null) for a product with no known public release-page pattern', () => { - seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: null }]) + it('derives the public package-page permalink for the product engine (private repo, never null)', () => { + seedRemote(remoteDir, 'brainy', [{ ...BASE_ENTRY, version: '11.0.5', url: 'https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.5' }]) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '11.0.6', date: '2026-09-03', bullets: ['fix: a native-only fix'] }])) const result = run( @@ -203,10 +203,20 @@ describe('wall-entry.mjs — generate + publish', () => { expect(result.status).toBe(0) const wall = readRemote(remoteDir, 'brainy') - expect(wall.entries[0].url).toBeNull() + expect(wall.entries[0].url).toBe('https://source.soulcraft.com/soulcraft/-/packages/npm/@soulcraft%2Fbrainy/11.0.6') expect(wall.entries[0].thumb).toBeNull() }) + it('refuses a product with no permalink pattern, naming the cure', () => { + seedRemote(remoteDir, 'open-brainy', [BASE_ENTRY]) + writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '1.0.0', date: '2026-09-03', bullets: ['feat: first'] }])) + + const result = run(['--product', 'mystery', '--version', '1.0.0', '--date', '2026-09-03', '--from-changelog', 'CHANGELOG.md', '--remote', remoteDir, '--cache-dir', cacheDir], dir) + expect(result.status).not.toBe(0) + expect(result.stderr).toMatch(/no permalink pattern for product "mystery"/) + expect(result.stderr).toMatch(/never carry url: null/) + }) + it('refuses when the CHANGELOG has no entry yet for the target version, and touches no remote', () => { seedRemote(remoteDir, 'open-brainy', []) writeFileSync(join(dir, 'CHANGELOG.md'), buildChangelog([{ version: '10.4.11', date: '2026-09-02', bullets: ['fix: whatever'] }])) From e435da787d79b85de6c0ef43c32ee40399e31860 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 15:41:11 -0700 Subject: [PATCH 136/150] =?UTF-8?q?fix(metadata):=20the=20indexable-array?= =?UTF-8?q?=20bound=20is=20256=20=E2=80=94=20a=20keyword=20list=20is=20not?= =?UTF-8?q?=20a=20vector?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 64 cleared tags, authors and labels, but not the shape that actually turns up in production metadata: a long keyword or participant list. 256 clears those and still refuses every embedding this engine will ever meet — the narrowest model it ships is 384-dimensional, so the two populations still do not overlap and nobody has to tune anything. A vector parked in metadata throws by name; a 200-keyword list writes and indexes. The number lives in ONE place, `MAX_INDEXED_ARRAY_LENGTH`, and every message, warning and pin derives it from there. Two pins still carried a literal: metadata-vector-exclusion refused an array of exactly 100 — which sits UNDER the new bound, so the case would have asserted a refusal that no longer happens — and the array-bound suite named "all 64 elements" in a title and picked its middle element as a hardcoded 't31'. Both derive from the constant now, so the pins follow it wherever it goes rather than silently inverting the next time it moves. --- src/errors/brainyError.ts | 13 +++++++++---- tests/integration/metadata-vector-exclusion.test.ts | 5 +++-- tests/unit/utils/metadataIndex-array-bound.test.ts | 12 +++++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/errors/brainyError.ts b/src/errors/brainyError.ts index 4301d3f7..2fbdbe8d 100644 --- a/src/errors/brainyError.ts +++ b/src/errors/brainyError.ts @@ -412,18 +412,23 @@ export class MigrationInProgressError extends BrainyError { * embedding parked in the metadata bag would mint 384 postings for one row. * The bound exists to keep that out of the index. * - * 64 is hardcoded on purpose (the zero-config law: no knob). It sits far above + * 256 is hardcoded on purpose (the zero-config law: no knob). It sits far above * every legitimate multi-value field the engine has seen — tags, authors, - * categories, labels, participant lists — and far below any real embedding - * width, so the two populations do not overlap and no caller has to tune it. + * categories, labels, keyword lists, participant lists — and still below the + * narrowest embedding this engine will ever meet (384 dimensions, the smallest + * model it ships), so the two populations do not overlap and no caller has to + * tune it. A vector parked in metadata is refused; a long keyword list is not. * * It replaces a limit of 10 that was applied SILENTLY: a row whose `tags` array * held eleven entries had that field skipped entirely and dropped out of every * filtered search on it, with no error, no warning and no way to tell the * difference from "no row matches". A rule this consequential is a law with a * name and a refusal, not a `continue`. + * + * This is the ONE place the number lives. Every message, warning, doc line and + * pin derives it from here — never a literal. */ -export const MAX_INDEXED_ARRAY_LENGTH = 64 +export const MAX_INDEXED_ARRAY_LENGTH = 256 /** * A metadata field carries an array longer than {@link MAX_INDEXED_ARRAY_LENGTH}. diff --git a/tests/integration/metadata-vector-exclusion.test.ts b/tests/integration/metadata-vector-exclusion.test.ts index 0ca25388..1943b215 100644 --- a/tests/integration/metadata-vector-exclusion.test.ts +++ b/tests/integration/metadata-vector-exclusion.test.ts @@ -161,7 +161,8 @@ describe('Metadata Vector Exclusion Fix', () => { // silence at a bound of 10 — the field simply vanished from the index and // the row dropped out of every `where` on it, indistinguishably from "no // row matches". The bound is now MAX_INDEXED_ARRAY_LENGTH and it REFUSES. - const largeArray = Array.from({ length: 100 }, (_, i) => `item${i}`) + const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1 + const largeArray = Array.from({ length: overTheBound }, (_, i) => `item${i}`) const err = await brainy .add({ @@ -176,7 +177,7 @@ describe('Metadata Vector Exclusion Fix', () => { expect(err).toBeInstanceOf(MetadataArrayTooLargeError) expect(err.field).toBe('items') - expect(err.length).toBe(100) + expect(err.length).toBe(overTheBound) expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) // Nothing was indexed from the refused write — no 'items' field, and above diff --git a/tests/unit/utils/metadataIndex-array-bound.test.ts b/tests/unit/utils/metadataIndex-array-bound.test.ts index cbbf6b63..a96ae1d6 100644 --- a/tests/unit/utils/metadataIndex-array-bound.test.ts +++ b/tests/unit/utils/metadataIndex-array-bound.test.ts @@ -15,9 +15,10 @@ * and the caller had no way to tell that from "no row matches". Eleven tags is * not an exotic shape; the eleventh tag made the row invisible. * - * THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH} = 64, + * THE LAW. Arrays of scalars index up to {@link MAX_INDEXED_ARRAY_LENGTH}, * hardcoded (the zero-config law: no knob), which clears every legitimate - * multi-value field and stays far below any embedding width. Above it the WRITE + * multi-value field — tags, authors, keyword lists — and stays below the + * narrowest embedding this engine meets (384 dimensions). Above it the WRITE * IS REFUSED by name — `MetadataArrayTooLargeError`, carrying the field, the * length and the bound — at `add`, `update`, `relate` and `updateRelation` * alike. Nothing is skipped in silence. @@ -65,7 +66,7 @@ describe('the indexable-array bound', () => { } }) - it('indexes right up to the bound — all 64 elements', async () => { + it('indexes right up to the bound — every element of it', async () => { await brain.add({ id: 'at-bound', data: 'a row at the bound', @@ -74,8 +75,9 @@ describe('the indexable-array bound', () => { vector: [] }) - // The first, the last, and one in the middle. - for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, 't31']) { + // The first, the last, and one in the middle — all derived from the + // bound, so the case follows the constant wherever it moves. + for (const tag of ['t0', `t${MAX_INDEXED_ARRAY_LENGTH - 1}`, `t${Math.floor(MAX_INDEXED_ARRAY_LENGTH / 2)}`]) { const hits = await brain.find({ where: { tags: tag }, limit: 10 } as any) expect(hits.map((r: any) => r.id)).toContain(resolveEntityId('at-bound')) } From d7444ae804853c57f1fc22328f56d3cca2e2ce3b Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 15:41:21 -0700 Subject: [PATCH 137/150] test(metadata): the three large-metadata cases pin the bound, not a magic length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get(), relate() and update() each carried a "very large metadata" case that parked an array of 1000 (or 100) elements in the metadata bag and asserted it came back. The indexable-array bound refuses that shape at the write door now — an array field mints one posting per element, so an unbounded array is an unbounded write — and the three cases were failing on the refusal they should have been pinning. Each is rewritten to the law that replaced it, in two halves: - a large SCALAR payload still round-trips whole through the door: a 10,000-character string, 100 sibling fields, a ten-deep nest walked to the bottom, and an array sitting exactly ON the bound, checked first element to last; - an array one element OVER the bound refuses with MetadataArrayTooLargeError carrying the field, the length and the bound, on the error object AND in the message. update()'s refusal additionally proves the row is unchanged, and relate()'s that no relation was written — refused means not written, not written-then-skipped. Every length is derived from the imported MAX_INDEXED_ARRAY_LENGTH; none is typed as a number. That is what made the old cases fragile: 100 read as "over the bound" and 1000 as "large", and both meanings changed under them when the constant moved. These follow the constant instead. --- tests/unit/brainy/get.test.ts | 62 +++++++++++++++++++++++----- tests/unit/brainy/relate.test.ts | 60 ++++++++++++++++++++++----- tests/unit/brainy/update.test.ts | 69 ++++++++++++++++++++++++++++---- 3 files changed, 165 insertions(+), 26 deletions(-) diff --git a/tests/unit/brainy/get.test.ts b/tests/unit/brainy/get.test.ts index b39bf2e1..97a19125 100644 --- a/tests/unit/brainy/get.test.ts +++ b/tests/unit/brainy/get.test.ts @@ -5,7 +5,8 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' -import { +import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError' +import { createAddParams, generateTestVector, createTestConfig, @@ -268,32 +269,75 @@ describe('Brainy.get()', () => { expect(entity!.id).toBe(id) }) - it('should get entity with very large metadata', async () => { - // Arrange + // THE INDEXABLE-ARRAY BOUND, from get()'s side. This case used to park a + // 1000-element array in the metadata bag and assert it came back. That + // shape is refused at the write door now — an array field mints one + // posting per element, so an unbounded array is an unbounded write — so + // the case pins BOTH halves of the law that replaced it: a large SCALAR + // payload still round-trips whole, and an array over the bound refuses by + // name. Every length derives from MAX_INDEXED_ARRAY_LENGTH so the pin + // follows the constant wherever it moves. + it('should get an entity with a large scalar metadata payload', async () => { + // Arrange — large in every dimension EXCEPT array length: a long string, + // many fields, deep nesting, and an array sitting exactly ON the bound. const largeMetadata = { - bigArray: new Array(1000).fill('item'), + atTheBound: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`), bigObject: Object.fromEntries( Array.from({ length: 100 }, (_, i) => [`key${i}`, `value${i}`]) ), + longString: 'x'.repeat(10_000), deepNesting: Array(10).fill(null).reduce( (acc) => ({ nested: acc }), { value: 'deep' } ) } - + const id = await brain.add(createAddParams({ data: 'Large metadata', type: 'thing', metadata: largeMetadata })) - + // Act const entity = await brain.get(id) - - // Assert + + // Assert — the payload comes back whole, first element to last expect(entity).not.toBeNull() - expect(entity!.metadata.bigArray).toHaveLength(1000) + expect(entity!.metadata.atTheBound).toHaveLength(MAX_INDEXED_ARRAY_LENGTH) + expect(entity!.metadata.atTheBound[0]).toBe('item0') + expect(entity!.metadata.atTheBound[MAX_INDEXED_ARRAY_LENGTH - 1]) + .toBe(`item${MAX_INDEXED_ARRAY_LENGTH - 1}`) expect(Object.keys(entity!.metadata.bigObject)).toHaveLength(100) + expect(entity!.metadata.longString).toHaveLength(10_000) + + // ...including the deep nest, walked to the bottom. + let cursor: any = entity!.metadata.deepNesting + for (let depth = 0; depth < 10; depth++) cursor = cursor.nested + expect(cursor.value).toBe('deep') + }) + + it('should refuse a metadata array over the indexing bound, by name', async () => { + // Arrange + const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1 + + // Act + const err = await brain + .add(createAddParams({ + data: 'Large metadata', + type: 'thing', + metadata: { bigArray: new Array(overTheBound).fill('item') } + })) + .catch((e: any) => e) + + // Assert — the field, the length and the bound, on the error and in the + // message, so a handler can report or repair without parsing prose. + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('bigArray') + expect(err.length).toBe(overTheBound) + expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) + expect(err.message).toContain('bigArray') + expect(err.message).toContain(String(overTheBound)) + expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) }) }) diff --git a/tests/unit/brainy/relate.test.ts b/tests/unit/brainy/relate.test.ts index eb1a036e..bea35ba3 100644 --- a/tests/unit/brainy/relate.test.ts +++ b/tests/unit/brainy/relate.test.ts @@ -5,7 +5,8 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' -import { +import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError' +import { createAddParams, createTestConfig, } from '../../helpers/test-factory' @@ -248,16 +249,23 @@ describe('Brainy.relate()', () => { expect(matches.length).toBe(1) // Only one relationship should exist }) - it('should handle very long metadata', async () => { - // Arrange + // THE INDEXABLE-ARRAY BOUND, from relate()'s side. This case used to pass a + // 100-element array through relate() and assert it came back — a length + // hardcoded either side of a bound it never named, so it read green or red + // purely by where the constant happened to sit. Both halves of the law are + // pinned here instead, and every length derives from + // MAX_INDEXED_ARRAY_LENGTH so the pin follows the constant. + it('should handle a large scalar metadata payload on a relation', async () => { + // Arrange — large in every dimension EXCEPT array length: a long string, + // many fields, and an array sitting exactly ON the bound. const largeMetadata = { - bigArray: new Array(100).fill('item'), + atTheBound: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`), bigObject: Object.fromEntries( Array.from({ length: 50 }, (_, i) => [`key${i}`, `value${i}`]) ), - longString: 'x'.repeat(1000) + longString: 'x'.repeat(10_000) } - + // Act await brain.relate({ from: entity1Id, @@ -265,12 +273,46 @@ describe('Brainy.relate()', () => { type: 'relatedTo', metadata: largeMetadata }) - - // Assert + + // Assert — the payload comes back whole, first element to last const relations = await brain.related({ from: entity1Id }) const relation = relations.find(r => r.to === entity2Id) expect(relation).toBeDefined() - expect(relation!.metadata?.bigArray).toHaveLength(100) + expect(relation!.metadata?.atTheBound).toHaveLength(MAX_INDEXED_ARRAY_LENGTH) + expect(relation!.metadata?.atTheBound[0]).toBe('item0') + expect(relation!.metadata?.atTheBound[MAX_INDEXED_ARRAY_LENGTH - 1]) + .toBe(`item${MAX_INDEXED_ARRAY_LENGTH - 1}`) + expect(Object.keys(relation!.metadata?.bigObject)).toHaveLength(50) + expect(relation!.metadata?.longString).toHaveLength(10_000) + }) + + it('should refuse a relation metadata array over the indexing bound, by name', async () => { + // Arrange + const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1 + + // Act + const err = await brain + .relate({ + from: entity1Id, + to: entity3Id, + type: 'relatedTo', + metadata: { bigArray: new Array(overTheBound).fill('item') } + }) + .catch((e: any) => e) + + // Assert — the field, the length and the bound, on the error and in the + // message, so a handler can report or repair without parsing prose. + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('bigArray') + expect(err.length).toBe(overTheBound) + expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) + expect(err.message).toContain('bigArray') + expect(err.message).toContain(String(overTheBound)) + expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) + + // Refused means not written: no relation of this shape exists. + const relations = await brain.related({ from: entity1Id }) + expect(relations.some(r => r.to === entity3Id && r.metadata?.bigArray)).toBe(false) }) it('should handle special characters in metadata', async () => { diff --git a/tests/unit/brainy/update.test.ts b/tests/unit/brainy/update.test.ts index 19fdad19..ec5f3fff 100644 --- a/tests/unit/brainy/update.test.ts +++ b/tests/unit/brainy/update.test.ts @@ -5,7 +5,8 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' -import { +import { MetadataArrayTooLargeError, MAX_INDEXED_ARRAY_LENGTH } from '../../../src/errors/brainyError' +import { createAddParams, createTestConfig, } from '../../helpers/test-factory' @@ -355,36 +356,88 @@ describe('Brainy.update()', () => { expect(final!.metadata.counter).toBeLessThanOrEqual(10) }) - it('should handle very large metadata updates', async () => { + // THE INDEXABLE-ARRAY BOUND, from update()'s side. This case used to write + // a 1000-element array through update() and assert it came back. That + // shape is refused at the write door now — an array field mints one + // posting per element, so an unbounded array is an unbounded write — so + // the case pins BOTH halves of the law that replaced it. Every length + // derives from MAX_INDEXED_ARRAY_LENGTH so the pin follows the constant. + it('should handle a large scalar metadata update', async () => { // Arrange const id = await brain.add(createAddParams({ data: 'Large metadata test', type: 'thing' })) - + + // Large in every dimension EXCEPT array length: a long string, many + // fields, deep nesting, and an array sitting exactly ON the bound. const largeMetadata = { - bigArray: new Array(1000).fill('item'), + atTheBound: Array.from({ length: MAX_INDEXED_ARRAY_LENGTH }, (_, i) => `item${i}`), bigObject: Object.fromEntries( Array.from({ length: 100 }, (_, i) => [`key${i}`, `value${i}`]) ), + longString: 'x'.repeat(10_000), deepNesting: Array(10).fill(null).reduce( (acc) => ({ nested: acc }), { value: 'deep' } ) } - + // Act await brain.update({ id, metadata: largeMetadata, merge: false }) - - // Assert + + // Assert — the payload comes back whole, first element to last const updated = await brain.get(id) expect(updated).not.toBeNull() - expect(updated!.metadata.bigArray).toHaveLength(1000) + expect(updated!.metadata.atTheBound).toHaveLength(MAX_INDEXED_ARRAY_LENGTH) + expect(updated!.metadata.atTheBound[0]).toBe('item0') + expect(updated!.metadata.atTheBound[MAX_INDEXED_ARRAY_LENGTH - 1]) + .toBe(`item${MAX_INDEXED_ARRAY_LENGTH - 1}`) expect(Object.keys(updated!.metadata.bigObject)).toHaveLength(100) + expect(updated!.metadata.longString).toHaveLength(10_000) + + // ...including the deep nest, walked to the bottom. + let cursor: any = updated!.metadata.deepNesting + for (let depth = 0; depth < 10; depth++) cursor = cursor.nested + expect(cursor.value).toBe('deep') + }) + + it('should refuse an update whose metadata array is over the indexing bound, by name', async () => { + // Arrange + const id = await brain.add(createAddParams({ + data: 'Large metadata test', + type: 'thing', + metadata: { keep: 'me' } + })) + const overTheBound = MAX_INDEXED_ARRAY_LENGTH + 1 + + // Act + const err = await brain + .update({ + id, + metadata: { bigArray: new Array(overTheBound).fill('item') }, + merge: false + }) + .catch((e: any) => e) + + // Assert — the field, the length and the bound, on the error and in the + // message, so a handler can report or repair without parsing prose. + expect(err).toBeInstanceOf(MetadataArrayTooLargeError) + expect(err.field).toBe('bigArray') + expect(err.length).toBe(overTheBound) + expect(err.limit).toBe(MAX_INDEXED_ARRAY_LENGTH) + expect(err.message).toContain('bigArray') + expect(err.message).toContain(String(overTheBound)) + expect(err.message).toContain(String(MAX_INDEXED_ARRAY_LENGTH)) + + // Refused means unchanged: the row still carries what it had before. + const unchanged = await brain.get(id) + expect(unchanged!.metadata.keep).toBe('me') + expect(unchanged!.metadata.bigArray).toBeUndefined() }) it('should preserve entity ID during update', async () => { From 87d3a945a53ec7470b8b72f70942ed4849176b75 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 15:53:23 -0700 Subject: [PATCH 138/150] test(idle): the idle pin says WHICH brain narrated, and asserts the attributable half first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "An idle brain prints nothing" is pinned two ways in this case, and only one of them is attributable. The spies are bound to THIS brain's providers, so they answer "did this brain flush?" exactly. The console filters cannot: the gate config runs the whole suite in ONE process (pool: 'forks', singleFork: true — verified, two files report the same process.pid), so console.log carries the narration of every brain alive in that process, including one a previous file opened and never closed whose unref'd cadence timer is still doing honest work. Ordered as it was, a neighbour's honest flush and this engine breaking its own law produced the same red, with a message that truncated the evidence to "[ …(4) ]" — no way to tell which had happened, and nothing to chase. So the spies assert first: their failure means the engine broke the law. The console assertion follows, keeps both patterns, and carries the captured lines in its message. vitest prefixes each stdout block with "stdout | > ", so the lines plus the surrounding log name the brain that printed them, and the next red is diagnosable from the log alone. No assertion is removed and no window is widened — the same two laws are pinned, in the order that makes a failure readable. --- tests/integration/idle-costs-nothing.test.ts | 29 ++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts index b5c386cf..2374627e 100644 --- a/tests/integration/idle-costs-nothing.test.ts +++ b/tests/integration/idle-costs-nothing.test.ts @@ -88,11 +88,36 @@ describe('an idle brain costs nothing', () => { } // (a) + (b): nothing ran, nothing was said. - expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) - expect(logged.filter((l) => /Flushing Brainy indexes/.test(l))).toEqual([]) + // + // THE SPIES COME FIRST, AND THEY ARE THE ATTRIBUTABLE HALF. They are bound + // to THIS brain's providers, so they answer "did this brain flush?" and + // nothing else. The console filters below cannot: the gate config runs the + // whole suite in ONE process (`pool: 'forks'`, `singleFork: true`), so + // `console.log` carries the narration of every brain alive in that + // process — including one a previous file opened and never closed, whose + // unref'd cadence timer is still doing honest work. A neighbour narrating + // is a REAL finding about suite hygiene, but it is not this brain failing + // its own law, and the two must not be reported as the same thing. + // + // So: spies first (whose failure means the engine broke the law), console + // second (whose failure means SOMETHING in the process narrated), and the + // console assertion carries the captured lines in its message. vitest's + // stdout blocks are prefixed `stdout | > `, so those lines + // plus the surrounding gate log name the brain that printed them. expect(countsSpy).not.toHaveBeenCalled() expect(metadataSpy).not.toHaveBeenCalled() expect(graphSpy).not.toHaveBeenCalled() + + const flushChatter = logged.filter( + (l) => /All indexes flushed to disk/.test(l) || /Flushing Brainy indexes/.test(l) + ) + expect( + flushChatter, + `a flush narrated during the ${IDLE_WATCH_MS}ms idle window. This brain's own ` + + `providers were NOT called (asserted above), so the lines below were printed by ` + + `another brain alive in this process — find it by the 'stdout | > ' ` + + `prefix in the run log:\n${flushChatter.join('\n')}` + ).toEqual([]) }, 180_000) it('an explicit flush over a clean brain calls no provider and prints nothing', async () => { From e766ed0a846a251e0807eb525f29829b56c3e2b0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 16:14:24 -0700 Subject: [PATCH 139/150] test(find-connected): close the brain this file leaks, and name the half a short answer came from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TWO THINGS, both about the same file. THE LEAK, which is a defect of the test. `afterAll` set `brain = null`. That does not close a brain — it only makes it unreachable from here. The instance stayed open and registered with its unref'd cadence timer running, and the gate config runs the whole suite in ONE process (pool: 'forks', singleFork: true — two files report the same process.pid), so a brain leaked in this file goes on narrating its flushes into every file that runs after it. This one holds 151 entities and 30 relations. It is closed now. It is not the only leaker in the suite — a create-versus-close scan turns up 67 files with the same shape, and this is one of them, not the cause of anything on its own. Fixing the file I was already in. THE DIAGNOSTIC. 'walks the vector leg over the neighbours only' went red on the gate box (1 row of a requested 5) while passing here in isolation eight runs out of eight, beside its own box predecessor, and under a perturbed random stream — and it passed on the box one gate earlier behind the IDENTICAL predecessor. So the cause is process state accumulated by the time this file runs, and a bare count mismatch says nothing about which half broke. The case now runs the same query without the vector leg first, as a control, and reports both counts: both short means the neighbour set or the filter, only the vector leg short means the walk — which matters here because every row in this corpus carries an IDENTICAL vector, so the walk is ranking an exact tie and a tie has no defined order to return 5 of. The assertion is unchanged: still exactly 5, still every row a neighbour. --- .../integration/find-connected-order.test.ts | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/integration/find-connected-order.test.ts b/tests/integration/find-connected-order.test.ts index b04e7f99..3b7560e4 100644 --- a/tests/integration/find-connected-order.test.ts +++ b/tests/integration/find-connected-order.test.ts @@ -67,6 +67,13 @@ describe('find({ connected }) is graph-first: neighbours → filter → page', ( }) afterAll(async () => { + // CLOSE IT. Dropping the reference does not close a brain — it only makes + // it unreachable from here. The instance stays open and registered, its + // unref'd cadence timer keeps running, and because the gate config runs the + // whole suite in ONE process (pool: 'forks', singleFork: true) it goes on + // narrating its flushes into every test file that runs after this one. + // A test that leaks a brain is a defect of the test. + await brain?.close() brain = null as any }) @@ -137,13 +144,34 @@ describe('find({ connected }) is graph-first: neighbours → filter → page', ( }) it('walks the vector leg over the neighbours only', async () => { + // The SAME query without the vector leg, first. Both legs draw from the + // one neighbour set, so this is the control: it says whether a short answer + // came from the adjacency/filter (both legs short) or from the vector walk + // alone (only the vector leg short). Cheap, and it turns a bare count + // mismatch into a named half — this case has gone red on the gate box + // while passing in isolation and beside its own predecessor, so the next + // red must arrive already carrying the half it belongs to. + const control = await brain.find({ + connected: { from: anchor, direction: 'out' }, + where: { kind: 'note' }, + limit: 5 + }) + const results = await brain.find({ vector: sharedVector, connected: { from: anchor, direction: 'out' }, where: { kind: 'note' }, limit: 5 }) - expect(results).toHaveLength(5) + + expect( + results.length, + `the vector leg returned ${results.length} of a requested 5. The same query ` + + `WITHOUT the vector returned ${control.length}: if that is also short the ` + + `neighbour set or the filter is the cause, and if it is 5 the vector walk is — ` + + `note every row in this corpus carries an identical vector, so the walk is ` + + `ranking an exact tie.` + ).toBe(5) for (const r of results) expect(neighbourIds.has(r.entity.id)).toBe(true) }) From dadfa61b5fd5baed5a6fcec100dcf54ef3f6dc3e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 2 Sep 2026 16:17:55 -0700 Subject: [PATCH 140/150] =?UTF-8?q?test(idle):=20capture=20the=20stack=20b?= =?UTF-8?q?ehind=20each=20flush=20narration=20=E2=80=94=20the=20line=20alo?= =?UTF-8?q?ne=20cannot=20name=20its=20brain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The self-diagnosis from the last round worked: the box says this brain's own providers were NOT called, so the flush pairs inside the 90 s window belong to another brain in the same process. It could not say WHICH, and the advice it gave — read the 'stdout | > ' prefix — cannot work here: vitest tags a stdout block with the test that is RUNNING, and these lines are captured by this test's own console hook anyway. Teeing them through would only ever print this test's name. The call stack does name the driver, so it is captured beside each line and the first one is reported: `kickBackgroundFlush('idle')` under `armIdleFlushTimer` is some brain's cadence timer, the deferred-embed worker's commit path is a brain still landing vectors, and a bare `flush()` is an explicit caller. Why that distinction settles it. A flush only narrates PAST the dirty gate, and `_dirtySinceLastFlush` is set in exactly three places — `noteWriteForPersistence()` (both commit paths, and the deferred-embed worker lands its vectors through the single-op one), `clear()`, and `repairIndex()`. So a narrating flush is a flush whose brain really did commit a write; "0 ms" is the flush being cheap, not the flush being empty. That reading rules OUT the re-arming-follow-up theory: the queued follow-up is armed only by a concurrent flush() caller, cleared before promotion, and a promoted run over a clean brain returns at the dirty gate without touching a provider or printing a line. Context the message now carries: the suite runs every file in ONE process, and a create-versus-close scan puts 67 test files above the line — more brains made than closed. This assertion is downstream of that, and the next red arrives with the stack that names which one. --- tests/integration/idle-costs-nothing.test.ts | 26 ++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts index 2374627e..7e664a28 100644 --- a/tests/integration/idle-costs-nothing.test.ts +++ b/tests/integration/idle-costs-nothing.test.ts @@ -70,8 +70,22 @@ describe('an idle brain costs nothing', () => { await brain.flush() const logged: string[] = [] + // The STACK behind each narration, kept beside the line it belongs to. + // vitest tags a stdout block with the test that is RUNNING, not the brain + // that wrote it, so teeing these lines through would only ever name this + // test. The call stack does name the driver: `kickBackgroundFlush('idle')` + // under `armIdleFlushTimer` is a cadence flush on some brain, the deferred- + // embed worker's commit path is a brain still landing vectors, and a bare + // `flush()` is an explicit caller. That distinction is the whole question. + const stacks: string[] = [] const origLog = console.log - console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + console.log = ((...a: unknown[]) => { + const line = a.map(String).join(' ') + logged.push(line) + if (/All indexes flushed to disk|Flushing Brainy indexes/.test(line)) { + stacks.push(new Error('flush narration').stack ?? '(no stack)') + } + }) as typeof console.log // Watch the providers directly: a flush that runs calls all of them. const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage @@ -113,10 +127,12 @@ describe('an idle brain costs nothing', () => { ) expect( flushChatter, - `a flush narrated during the ${IDLE_WATCH_MS}ms idle window. This brain's own ` + - `providers were NOT called (asserted above), so the lines below were printed by ` + - `another brain alive in this process — find it by the 'stdout | > ' ` + - `prefix in the run log:\n${flushChatter.join('\n')}` + `${flushChatter.length} flush line(s) narrated during the ${IDLE_WATCH_MS}ms idle ` + + `window. This brain's own providers were NOT called (asserted above), so another ` + + `brain alive in this process printed them — the suite runs every file in ONE ` + + `process and 67 test files create more brains than they close.\n` + + `${flushChatter.join('\n')}\n\n` + + `The stack behind the first one names the driver:\n${stacks[0] ?? '(none captured)'}` ).toEqual([]) }, 180_000) From 4c344782a75d686b878f0e3f2522c516efaceb67 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:06:00 -0700 Subject: [PATCH 141/150] test(hygiene): close every brain the find suite creates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/integration/find-*.test.ts and tests/unit/brainy/find*.test.ts each opened one or more Brainy instances (via beforeAll/beforeEach) and never closed them — the leaked instance's cadence timer stays armed for the rest of the single-forked vitest run and keeps narrating into every later file. find-unified-integration.test.ts was a real bug, not just a missing hook: its afterAll called a no-op TestCleanup().cleanup() (nothing was ever registered with it) and then discarded the brain reference with `brain = null` — the brain was never actually closed. --- tests/integration/find-fields-projection.test.ts | 6 +++++- tests/integration/find-near.test.ts | 6 +++++- tests/integration/find-orderby-every-path.test.ts | 6 +++++- tests/integration/find-planner-door.test.ts | 6 +++++- tests/integration/find-unified-integration.test.ts | 1 + tests/unit/brainy/find-complement-operators.test.ts | 6 +++++- tests/unit/brainy/find-index-integrity-guard.test.ts | 6 +++++- tests/unit/brainy/find.test.ts | 8 ++++++-- 8 files changed, 37 insertions(+), 8 deletions(-) diff --git a/tests/integration/find-fields-projection.test.ts b/tests/integration/find-fields-projection.test.ts index 5d339f08..25ee416c 100644 --- a/tests/integration/find-fields-projection.test.ts +++ b/tests/integration/find-fields-projection.test.ts @@ -19,7 +19,7 @@ * index-served (a body field, or a bucketed timestamp), exactly the owing rows * are read and the rest are still served from the index. */ -import { describe, it, expect, beforeAll, vi } from 'vitest' +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType } from '../../src/types/graphTypes' import { generateTestVector } from '../helpers/test-factory' @@ -59,6 +59,10 @@ describe('find/get({ fields }) — projection', () => { await brain.flush() }) + afterAll(async () => { + await brain.close() + }) + /** Count canonical record reads for one call. */ const countingReads = async (body: () => Promise): Promise<{ out: R; reads: number }> => { const spy = vi.spyOn(brain as any, 'batchGet') diff --git a/tests/integration/find-near.test.ts b/tests/integration/find-near.test.ts index 3fb235c8..b2bf01cd 100644 --- a/tests/integration/find-near.test.ts +++ b/tests/integration/find-near.test.ts @@ -9,7 +9,7 @@ * it). Now the anchor is fetched with its vector, and an anchor without one * refuses by name instead of failing inside the index. */ -import { describe, it, expect, beforeAll } from 'vitest' +import { describe, it, expect, beforeAll, afterAll } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType } from '../../src/types/graphTypes' import { v5 } from '../../src/universal/uuid' @@ -28,6 +28,10 @@ describe('find({ near }) uses the anchor vector', () => { await brain.add({ id: 'far', data: 'far row', type: NounType.Thing, vector: generateTestVector() }) }) + afterAll(async () => { + await brain.close() + }) + it('returns the anchor\'s neighbours by its own vector', async () => { const results = await brain.find({ near: { id: 'anchor' }, limit: 3 }) expect(results.length).toBeGreaterThan(0) diff --git a/tests/integration/find-orderby-every-path.test.ts b/tests/integration/find-orderby-every-path.test.ts index 7637a79b..e62ec670 100644 --- a/tests/integration/find-orderby-every-path.test.ts +++ b/tests/integration/find-orderby-every-path.test.ts @@ -40,7 +40,7 @@ * the covering is ASSERTED from the leg's own output rather than assumed. This * pin is about ordering, and it says nothing about recall. */ -import { describe, it, expect, beforeAll } from 'vitest' +import { describe, it, expect, beforeAll, afterAll } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType, VerbType } from '../../src/types/graphTypes' import { resolveEntityId } from '../../src/utils/idNormalization' @@ -107,6 +107,10 @@ describe('find(): orderBy is the order on every path', () => { } }) + afterAll(async () => { + await brain.close() + }) + it('the fixture: the hybrid candidate set covers the whole filter universe', async () => { const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) expect(universe).toHaveLength(ROWS) diff --git a/tests/integration/find-planner-door.test.ts b/tests/integration/find-planner-door.test.ts index 964b13f9..e5224f6d 100644 --- a/tests/integration/find-planner-door.test.ts +++ b/tests/integration/find-planner-door.test.ts @@ -23,7 +23,7 @@ * against the adjacency before it is believed, so a not-serving graph refuses * loudly instead of answering `[]` as truth. */ -import { describe, it, expect, beforeAll, vi } from 'vitest' +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType, VerbType } from '../../src/types/graphTypes' import { generateTestVector } from '../helpers/test-factory' @@ -56,6 +56,10 @@ describe('find(): the optional planner door', () => { } }) + afterAll(async () => { + await brain.close() + }) + /** Install a planner door for one call, then remove it. */ const withDoor = async ( door: (...a: any[]) => Promise, diff --git a/tests/integration/find-unified-integration.test.ts b/tests/integration/find-unified-integration.test.ts index 94053d55..3c4741c2 100644 --- a/tests/integration/find-unified-integration.test.ts +++ b/tests/integration/find-unified-integration.test.ts @@ -48,6 +48,7 @@ describe('Unified Find() Integration Tests', () => { afterAll(async () => { await cleanup.cleanup() + await brain.close() brain = null as any }) diff --git a/tests/unit/brainy/find-complement-operators.test.ts b/tests/unit/brainy/find-complement-operators.test.ts index 76fbb017..710fbbbf 100644 --- a/tests/unit/brainy/find-complement-operators.test.ts +++ b/tests/unit/brainy/find-complement-operators.test.ts @@ -7,7 +7,7 @@ * soft-delete semantic: `field !== value` MUST include entities that have no * such field at all. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' @@ -26,6 +26,10 @@ describe('find() complement operators (ne / exists:false / missing:true)', () => ids.noField2 = await brain.add({ data: 'n2', type: NounType.Thing, metadata: { other: 2 } }) }) + afterEach(async () => { + await brain.close() + }) + it('ne returns everything except the matching value — INCLUDING entities without the field', async () => { const rows = await brain.find({ where: { status: { ne: 'active' } }, limit: 100 }) const got = new Set(rows.map((r) => r.id)) diff --git a/tests/unit/brainy/find-index-integrity-guard.test.ts b/tests/unit/brainy/find-index-integrity-guard.test.ts index 30cfdf1b..3e63d790 100644 --- a/tests/unit/brainy/find-index-integrity-guard.test.ts +++ b/tests/unit/brainy/find-index-integrity-guard.test.ts @@ -12,7 +12,7 @@ * returns an id whose record matches NEITHER the type nor the where filter) and * assert the phantom is dropped while the genuine matches survive. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' @@ -48,6 +48,10 @@ describe('find() index-integrity guard (phantom row class)', () => { }) }) + afterEach(async () => { + await brain.close() + }) + it('healthy index: the discriminant query returns only the staff Person', async () => { const rows = await brain.find({ type: NounType.Person, where: { entityType: 'staff' }, limit: 100 }) expect(rows.map((r) => r.id)).toEqual([staffId]) diff --git a/tests/unit/brainy/find.test.ts b/tests/unit/brainy/find.test.ts index 5bead272..59601456 100644 --- a/tests/unit/brainy/find.test.ts +++ b/tests/unit/brainy/find.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { createAddParams } from '../../helpers/test-factory' import { NounType } from '../../../src/types/graphTypes' @@ -12,7 +12,11 @@ describe('Brainy.find()', () => { }) await brain.init() }) - + + afterEach(async () => { + await brain.close() + }) + describe('success paths', () => { it('should find entities by text query', async () => { // Arrange From d6e7453f1f67ec264a1c5bf565246bf11dbf9235 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:06:04 -0700 Subject: [PATCH 142/150] test(hygiene): close every brain the integration suite creates Each file opened a Brainy in beforeAll/beforeEach (or a single it()) and never closed it. related-verb-array.test.ts and vfs-containment-batched.test.ts were real bugs: their afterAll discarded the brain with `brain = null as any` without ever calling close() first. --- tests/integration/api-parameter-validation.test.ts | 4 ++++ tests/integration/entity-confidence-weight.test.ts | 6 +++++- tests/integration/related-verb-array.test.ts | 1 + tests/integration/relationship-intelligence.test.ts | 3 ++- tests/integration/rev-and-ifabsent.test.ts | 6 +++++- tests/integration/vfs-containment-batched.test.ts | 1 + tests/integration/vfs-debug.test.ts | 8 ++++++-- 7 files changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/integration/api-parameter-validation.test.ts b/tests/integration/api-parameter-validation.test.ts index 4e25e781..da4aed14 100644 --- a/tests/integration/api-parameter-validation.test.ts +++ b/tests/integration/api-parameter-validation.test.ts @@ -34,6 +34,10 @@ describe('API Parameter Validation', () => { }) }) + afterAll(async () => { + await brain.close() + }) + it('should use "where" parameter for metadata filtering', async () => { const results = await brain.find({ where: { category: 'test-category' }, diff --git a/tests/integration/entity-confidence-weight.test.ts b/tests/integration/entity-confidence-weight.test.ts index b5bb34c5..031d29f1 100644 --- a/tests/integration/entity-confidence-weight.test.ts +++ b/tests/integration/entity-confidence-weight.test.ts @@ -7,7 +7,7 @@ * - Backward compatibility preserved */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' @@ -19,6 +19,10 @@ describe('Entity Confidence & Weight Exposure', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + describe('Entity interface', () => { it('should expose confidence when adding entity with confidence', async () => { const id = await brain.add({ diff --git a/tests/integration/related-verb-array.test.ts b/tests/integration/related-verb-array.test.ts index 36a49850..7ed1bd3f 100644 --- a/tests/integration/related-verb-array.test.ts +++ b/tests/integration/related-verb-array.test.ts @@ -30,6 +30,7 @@ describe('related() with a verb-type array returns every requested type', () => }) afterAll(async () => { + await brain.close() brain = null as any }) diff --git a/tests/integration/relationship-intelligence.test.ts b/tests/integration/relationship-intelligence.test.ts index b6e11cb5..c18057fb 100644 --- a/tests/integration/relationship-intelligence.test.ts +++ b/tests/integration/relationship-intelligence.test.ts @@ -59,7 +59,8 @@ describe('Relationship Intelligence', () => { await brain.init() }) - afterEach(() => { + afterEach(async () => { + await brain.close() if (fs.existsSync(testDir)) { fs.rmSync(testDir, { recursive: true }) } diff --git a/tests/integration/rev-and-ifabsent.test.ts b/tests/integration/rev-and-ifabsent.test.ts index 64b184a3..3bff59f1 100644 --- a/tests/integration/rev-and-ifabsent.test.ts +++ b/tests/integration/rev-and-ifabsent.test.ts @@ -9,7 +9,7 @@ * - addMany({ ifAbsent: true }) applies the flag to every item */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { RevisionConflictError } from '../../src/transaction/RevisionConflictError.js' import { NounType } from '../../src/types/graphTypes.js' @@ -22,6 +22,10 @@ describe('7.31.0 — _rev CAS + ifAbsent', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + describe('_rev initialization + surface', () => { it('initializes _rev to 1 on add()', async () => { const id = await brain.add({ data: 'hello', type: NounType.Document }) diff --git a/tests/integration/vfs-containment-batched.test.ts b/tests/integration/vfs-containment-batched.test.ts index 0a7919bf..7bbad478 100644 --- a/tests/integration/vfs-containment-batched.test.ts +++ b/tests/integration/vfs-containment-batched.test.ts @@ -81,6 +81,7 @@ describe('repairContainment: batched pass 2', () => { }) afterAll(async () => { + await brain.close() brain = null as any }) diff --git a/tests/integration/vfs-debug.test.ts b/tests/integration/vfs-debug.test.ts index 7e781139..5eeb0ef5 100644 --- a/tests/integration/vfs-debug.test.ts +++ b/tests/integration/vfs-debug.test.ts @@ -9,9 +9,10 @@ import * as XLSX from 'xlsx' describe('VFS Debug', () => { it('minimal VFS writeFile test', async () => { const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() + try { + await brain.init() - console.log('✅ Brain initialized') + console.log('✅ Brain initialized') // Get VFS and initialize const vfs = brain.vfs @@ -77,5 +78,8 @@ describe('VFS Debug', () => { // THE REAL TEST: Can we query VFS? expect(children.length).toBeGreaterThan(0) expect(rootContents.length).toBeGreaterThan(0) + } finally { + await brain.close() + } }) }) From de79d6b5a4ca41701c23a8b0b235a45029737780 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:06:10 -0700 Subject: [PATCH 143/150] test(hygiene): close every brain the unit suite creates Each file opened one or more Brainy instances (beforeEach, or a small per-test helper like migration-gate-family-scoped's module-level seed()) and never closed them. migration-gate-family-scoped.test.ts now tracks every brain seed() hands back in a describe-scoped array drained by afterEach, since the helper itself lives outside the describe block. --- tests/unit/brainy-core.unit.test.ts | 6 +++++- tests/unit/brainy/metadata-provider-contract.test.ts | 6 +++++- .../unit/brainy/migration-gate-family-scoped.test.ts | 12 +++++++++++- .../brainy/relate-duplicate-optimization.test.ts | 2 +- tests/unit/get-index-status-readiness.test.ts | 6 +++++- .../graph/graph-fastpath-honest-readiness.test.ts | 6 +++++- tests/unit/metadata-cold-read-guard.test.ts | 6 +++++- tests/unit/migration-lock.test.ts | 11 ++++++++++- tests/unit/neural/signals/EmbeddingSignal.test.ts | 3 ++- .../storage/pagination-parallel-hydration.test.ts | 6 +++++- tests/unit/type-filtering.unit.test.ts | 6 +++++- tests/unit/utils/metadataIndex-array-bound.test.ts | 4 ++++ .../metadataIndex-sparse-range-collation.test.ts | 6 +++++- tests/unit/validate-invariants-delegation.test.ts | 6 +++++- tests/unit/vector-cold-read-guard.test.ts | 6 +++++- tests/unit/vfs-multi-instance-diagnostic.test.ts | 6 +++++- 16 files changed, 83 insertions(+), 15 deletions(-) diff --git a/tests/unit/brainy-core.unit.test.ts b/tests/unit/brainy-core.unit.test.ts index eb6614e4..0488057d 100644 --- a/tests/unit/brainy-core.unit.test.ts +++ b/tests/unit/brainy-core.unit.test.ts @@ -5,7 +5,7 @@ * No mocks, no fakes, real implementation */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' @@ -21,6 +21,10 @@ describe('Brainy 3.0 Core (Unit Tests)', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + describe('CRUD Operations', () => { it('should create items with add', async () => { const id = await brain.add({ diff --git a/tests/unit/brainy/metadata-provider-contract.test.ts b/tests/unit/brainy/metadata-provider-contract.test.ts index 945c0670..466fc654 100644 --- a/tests/unit/brainy/metadata-provider-contract.test.ts +++ b/tests/unit/brainy/metadata-provider-contract.test.ts @@ -18,7 +18,7 @@ * exercised by cor's combined matrix); they inject probe/spy hooks onto the live JS * metadata index, which has neither method by default. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' @@ -34,6 +34,10 @@ describe('metadata-provider contract wiring (getIdsForFilter opts)', () => { mi = (brain as any).metadataIndex }) + afterEach(async () => { + await brain.close() + }) + it('RETIRED: a read never calls probeConsistency() / self-heals via detectAndRepairCorruption — that is the read-triggered dark rebuild the health-gate law forbids', async () => { let probes = 0 let repairs = 0 diff --git a/tests/unit/brainy/migration-gate-family-scoped.test.ts b/tests/unit/brainy/migration-gate-family-scoped.test.ts index b71c3899..ce510a4e 100644 --- a/tests/unit/brainy/migration-gate-family-scoped.test.ts +++ b/tests/unit/brainy/migration-gate-family-scoped.test.ts @@ -8,7 +8,7 @@ * gate that hung getStats / readdir / readFile behind an unrelated family's * migration until the wait timed out. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy.js' import { MigrationInProgressError } from '../../../src/errors/brainyError.js' @@ -38,12 +38,19 @@ const jam = (provider: unknown) => { } describe('migration LOCK is family-scoped', () => { + const opened: Brainy[] = [] + beforeEach(() => { process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' }) + afterEach(async () => { + for (const b of opened.splice(0)) await b.close().catch(() => {}) + }) + it('a stuck VECTOR migration does not block canonical or graph/metadata reads', async () => { const brain = await seed() + opened.push(brain) const childId = ( (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> )[0].entityId @@ -60,6 +67,7 @@ describe('migration LOCK is family-scoped', () => { it('a stuck VECTOR migration STILL blocks a read that needs the vector family', async () => { const brain = await seed() + opened.push(brain) jam((brain as any).index) // A semantic query consults the vector index — it must wait, and (bounded by @@ -70,6 +78,7 @@ describe('migration LOCK is family-scoped', () => { it('a stuck GRAPH migration blocks traversal but not vector/canonical reads', async () => { const brain = await seed() + opened.push(brain) const childId = ( (await brain.vfs.readdir('/notes', { withFileTypes: true })) as Array<{ entityId: string }> )[0].entityId @@ -87,6 +96,7 @@ describe('migration LOCK is family-scoped', () => { it('with no migration in flight, every read serves (the fast path is a no-op)', async () => { const brain = await seed() + opened.push(brain) await expect(brain.getStats()).resolves.toBeDefined() await expect(brain.find({ query: 'doc' })).resolves.toBeDefined() await expect(brain.vfs.readdir('/notes')).resolves.toHaveLength(1) diff --git a/tests/unit/brainy/relate-duplicate-optimization.test.ts b/tests/unit/brainy/relate-duplicate-optimization.test.ts index 8bcb7c7a..910d057d 100644 --- a/tests/unit/brainy/relate-duplicate-optimization.test.ts +++ b/tests/unit/brainy/relate-duplicate-optimization.test.ts @@ -18,7 +18,7 @@ describe('Duplicate Check Optimization', () => { }) afterEach(async () => { - // Cleanup is automatic with memory storage + await brain.close() }) it('should detect duplicate relationships using GraphAdjacencyIndex', async () => { diff --git a/tests/unit/get-index-status-readiness.test.ts b/tests/unit/get-index-status-readiness.test.ts index 7f82ec5d..5e283bc8 100644 --- a/tests/unit/get-index-status-readiness.test.ts +++ b/tests/unit/get-index-status-readiness.test.ts @@ -7,7 +7,7 @@ * _indexRebuildFailed / _indexDegradedIds degraded states (mirroring * validateIndexConsistency / checkHealth). */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType } from '../../src/index.js' describe('getIndexStatus honest readiness (Finding 9)', () => { @@ -20,6 +20,10 @@ describe('getIndexStatus honest readiness (Finding 9)', () => { await brain.flush() }) + afterEach(async () => { + await brain.close() + }) + it('a not-ready provider makes populated honest (false) and exposes ready:false', async () => { brain.index.isReady = () => false // count present, serving structure NOT loaded const status = await brain.getIndexStatus() diff --git a/tests/unit/graph/graph-fastpath-honest-readiness.test.ts b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts index 46d318b4..95a6c0c4 100644 --- a/tests/unit/graph/graph-fastpath-honest-readiness.test.ts +++ b/tests/unit/graph/graph-fastpath-honest-readiness.test.ts @@ -8,7 +8,7 @@ * scan; and a one-shot probe self-heals a no-isReady provider whose adjacency * did not cold-load. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType, VerbType } from '../../../src/index.js' describe('graph fast-path honest readiness (Finding 2)', () => { @@ -33,6 +33,10 @@ describe('graph fast-path honest readiness (Finding 2)', () => { await storage.getVerbsBySource(a) }) + afterEach(async () => { + await brain.close() + }) + it('not-ready provider → shard scan returns the REAL edges, not a silent []', async () => { const gi = storage.graphIndex // Simulate a cold native provider: count/manifest loaded (isInitialized) but diff --git a/tests/unit/metadata-cold-read-guard.test.ts b/tests/unit/metadata-cold-read-guard.test.ts index b4f82f15..d079982e 100644 --- a/tests/unit/metadata-cold-read-guard.test.ts +++ b/tests/unit/metadata-cold-read-guard.test.ts @@ -15,7 +15,7 @@ * The 8.0 JS index cold-loads correctly, so we simulate the cold native failure * mode by intercepting the provider's getIdsForFilter/rebuild. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType, MetadataIndexNotReadyError } from '../../src/index.js' const V = () => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001) @@ -31,6 +31,10 @@ describe('Metadata cold-read guard (#venue silent-[])', () => { await brain.flush() }) + afterEach(async () => { + await brain.close() + }) + it('warm brain: filtered find is correct and the guard does not rebuild', async () => { const mi = brain.metadataIndex let rebuilds = 0 diff --git a/tests/unit/migration-lock.test.ts b/tests/unit/migration-lock.test.ts index f0fbbe4c..63f6953e 100644 --- a/tests/unit/migration-lock.test.ts +++ b/tests/unit/migration-lock.test.ts @@ -18,7 +18,7 @@ * the production feature-detection reads it. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType, MigrationInProgressError } from '../../src/index.js' import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js' @@ -39,6 +39,12 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { await brain.init() }) + afterEach(async () => { + // The "close() is not gated" test already closes `brain` itself as its + // own assertion — closing an already-closed brain is a safe no-op here. + await brain.close().catch(() => {}) + }) + it('does not gate operations when no provider is migrating (fast path)', async () => { const id = await brain.add({ data: 'hello', type: NounType.Concept }) expect(id).toBeTruthy() @@ -130,6 +136,9 @@ describe('Migration LOCK (#18) — coordinated 7.x→8.0 auto-upgrade', () => { expect(e).toBeInstanceOf(MigrationInProgressError) expect(e.retryable).toBe(true) expect(typeof e.elapsedMs).toBe('number') + } finally { + // close() is proven not-gated by the test below — safe even mid-migration. + await shortBrain.close() } }) diff --git a/tests/unit/neural/signals/EmbeddingSignal.test.ts b/tests/unit/neural/signals/EmbeddingSignal.test.ts index 54d34b64..ad08e045 100644 --- a/tests/unit/neural/signals/EmbeddingSignal.test.ts +++ b/tests/unit/neural/signals/EmbeddingSignal.test.ts @@ -13,10 +13,11 @@ describe('EmbeddingSignal', () => { signal = new EmbeddingSignal(brain) }) - afterEach(() => { + afterEach(async () => { signal.clearCache() signal.clearHistory() signal.resetStats() + await brain.close() }) describe('initialization', () => { diff --git a/tests/unit/storage/pagination-parallel-hydration.test.ts b/tests/unit/storage/pagination-parallel-hydration.test.ts index ada324bb..a98fe8c9 100644 --- a/tests/unit/storage/pagination-parallel-hydration.test.ts +++ b/tests/unit/storage/pagination-parallel-hydration.test.ts @@ -7,7 +7,7 @@ * hydration (zero per-entity reads when unfiltered). Both must preserve the exact * pagination contract: same order, cursor continuation, filters, totalCount. */ -import { describe, it, expect, beforeEach, vi } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { Brainy, NounType } from '../../../src/index.js' describe('paginated enumeration — parallel hydration + id-only (cortex heal-cost)', () => { @@ -30,6 +30,10 @@ describe('paginated enumeration — parallel hydration + id-only (cortex heal-co storage = brain.storage }) + afterEach(async () => { + await brain.close() + }) + /** Page the whole dataset through a small limit via cursor and collect ordered ids. */ const pageAll = async (fn: (opts: any) => Promise, key: 'items' | 'ids') => { const out: string[] = [] diff --git a/tests/unit/type-filtering.unit.test.ts b/tests/unit/type-filtering.unit.test.ts index 9e4700b2..a1943da9 100644 --- a/tests/unit/type-filtering.unit.test.ts +++ b/tests/unit/type-filtering.unit.test.ts @@ -4,7 +4,7 @@ * Tests to verify that brain.find({ type: NounType.X }) correctly filters entities */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType } from '../../src/index.js' describe('Type Filtering (A Consumer Team Issue)', () => { @@ -17,6 +17,10 @@ describe('Type Filtering (A Consumer Team Issue)', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + it('should filter entities by NounType.Person', async () => { // Add 3 people await brain.add({ data: 'John Smith', type: NounType.Person, metadata: { name: 'John' } }) diff --git a/tests/unit/utils/metadataIndex-array-bound.test.ts b/tests/unit/utils/metadataIndex-array-bound.test.ts index a96ae1d6..32bf5d8c 100644 --- a/tests/unit/utils/metadataIndex-array-bound.test.ts +++ b/tests/unit/utils/metadataIndex-array-bound.test.ts @@ -48,6 +48,10 @@ describe('the indexable-array bound', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + describe('BELOW the bound: the array indexes, every element of it', () => { it('the eleven-element array that used to vanish is searchable', async () => { // ELEVEN — one over the old silent limit, the whole shape of the defect. diff --git a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts index d6d00568..7a2bf0a7 100644 --- a/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts +++ b/tests/unit/utils/metadataIndex-sparse-range-collation.test.ts @@ -42,7 +42,7 @@ * column store adopts the field. It is named in `getIdsFromChunksForRange`'s * doc comment rather than papered over. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../../src/brainy' import { NounType } from '../../../src/types/graphTypes' import { SparseIndex, ChunkManager } from '../../../src/utils/metadataIndexChunking' @@ -122,6 +122,10 @@ describe('legacy sparse index: range queries order values, or refuse', () => { expect(index.columnStore.hasField(FIELD)).toBe(false) }) + afterEach(async () => { + await brain.close() + }) + describe('(a) a long BOUND against ordinary short values', () => { // 'apple' < 'mango' < 'zebra', and every bound below is compared against // these three raw keys. diff --git a/tests/unit/validate-invariants-delegation.test.ts b/tests/unit/validate-invariants-delegation.test.ts index a5def81f..69133733 100644 --- a/tests/unit/validate-invariants-delegation.test.ts +++ b/tests/unit/validate-invariants-delegation.test.ts @@ -6,7 +6,7 @@ * validateInvariants(), and repairIndex() maps a failing invariant with heal:'rebuild' * to that provider's rebuild(). "healthy-while-broken must be impossible." */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType } from '../../src/index.js' import type { ProviderInvariantReport } from '../../src/index.js' @@ -48,6 +48,10 @@ describe('validateIndexConsistency delegates to provider validateInvariants() (P await brain.flush() }) + afterEach(async () => { + await brain.close() + }) + it('a broken provider report makes the store unhealthy and names the failing invariant', async () => { brain.index.validateInvariants = async () => brokenReport('vector') const v = await brain.validateIndexConsistency() diff --git a/tests/unit/vector-cold-read-guard.test.ts b/tests/unit/vector-cold-read-guard.test.ts index 0905f298..963009b7 100644 --- a/tests/unit/vector-cold-read-guard.test.ts +++ b/tests/unit/vector-cold-read-guard.test.ts @@ -12,7 +12,7 @@ * signal (from either strategy) THROWS VectorIndexNotReadyError immediately, * with no rebuild attempt in between — never a silent empty result. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType, VectorIndexNotReadyError } from '../../src/index.js' const V = (): number[] => Array.from({ length: 384 }, (_, i) => Math.sin(i * 0.1) + 0.001) @@ -28,6 +28,10 @@ describe('Vector cold-read guard (verifyVectorLive) — silent-[] on cold semant await brain.flush() }) + afterEach(async () => { + await brain.close() + }) + it('warm brain: semantic find is correct and the guard does not rebuild', async () => { const vi = brain.index let rebuilds = 0 diff --git a/tests/unit/vfs-multi-instance-diagnostic.test.ts b/tests/unit/vfs-multi-instance-diagnostic.test.ts index deaa4615..85ff1002 100644 --- a/tests/unit/vfs-multi-instance-diagnostic.test.ts +++ b/tests/unit/vfs-multi-instance-diagnostic.test.ts @@ -4,7 +4,7 @@ * Tests to verify VFS import behavior and identify if VFS creates only wrappers or also graph entities */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy, NounType } from '../../src/index.js' describe('VFS Multi-instance Diagnostic', () => { @@ -17,6 +17,10 @@ describe('VFS Multi-instance Diagnostic', () => { await brain.init() }) + afterEach(async () => { + await brain.close() + }) + it('should verify VFS creates document wrappers AND allows entity filtering', async () => { console.log('\n🔬 VFS Multi-instance Diagnostic Test\n') console.log('='.repeat(70)) From be307a15794246e95799123d9515e1ade0cedf56 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:06:13 -0700 Subject: [PATCH 144/150] test(hygiene): close every brain the vfs unit suite creates Each file opened a Brainy per test (beforeEach) and never closed it. --- tests/vfs/tree-operations.unit.test.ts | 6 +++++- tests/vfs/vfs-bug-fixes.unit.test.ts | 6 +++++- tests/vfs/vfs-bulkwrite-race.unit.test.ts | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/vfs/tree-operations.unit.test.ts b/tests/vfs/tree-operations.unit.test.ts index 8c717115..91743227 100644 --- a/tests/vfs/tree-operations.unit.test.ts +++ b/tests/vfs/tree-operations.unit.test.ts @@ -3,7 +3,7 @@ * Ensures tree methods prevent recursion and work correctly */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' import { VFSTreeUtils } from '../../src/vfs/TreeUtils.js' @@ -24,6 +24,10 @@ describe('VFS Tree Operations', () => { await vfs.init() }) + afterEach(async () => { + await brain.close() + }) + describe('Critical: No Self-Inclusion Bug', () => { it('should NEVER return a directory as its own child', async () => { // Create test structure diff --git a/tests/vfs/vfs-bug-fixes.unit.test.ts b/tests/vfs/vfs-bug-fixes.unit.test.ts index f98d6a76..12199c8b 100644 --- a/tests/vfs/vfs-bug-fixes.unit.test.ts +++ b/tests/vfs/vfs-bug-fixes.unit.test.ts @@ -6,7 +6,7 @@ * - Issue #2: File read decompression error */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' @@ -25,6 +25,10 @@ describe('VFS Bug Fixes', () => { await vfs.init() }) + afterEach(async () => { + await brain.close() + }) + describe('Issue #1: Duplicate Directory Nodes', () => { it('should not create duplicate directory entries when writing multiple files to same directory', async () => { // Write multiple files to the same directory (reproduce the bug scenario) diff --git a/tests/vfs/vfs-bulkwrite-race.unit.test.ts b/tests/vfs/vfs-bulkwrite-race.unit.test.ts index 238ac6b9..09d68568 100644 --- a/tests/vfs/vfs-bulkwrite-race.unit.test.ts +++ b/tests/vfs/vfs-bulkwrite-race.unit.test.ts @@ -12,7 +12,7 @@ * other operations in parallel batches. */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { VirtualFileSystem } from '../../src/vfs/VirtualFileSystem.js' @@ -30,6 +30,10 @@ describe('VFS bulkWrite Race Condition Fix', () => { await vfs.init() }) + afterEach(async () => { + await brain.close() + }) + describe('operation ordering', () => { it('should create directories before files when mixed in same batch', async () => { // This is the exact scenario that triggered the race condition: From 4e058720b43dcfb469b2823b218673b28be711ea Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 08:52:58 -0700 Subject: [PATCH 145/150] fix(index): a field holds every value kind it was written with, not the first one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metadata index fixed a field's value type from the first value it saw. Every later value of another kind was coerced to that type, and when coercion failed — `Number('electronics')` is NaN — the value was dropped from the index with no error at all. The row stayed readable by id and by vector search and vanished only from equality filters on that one field, which is what made it so quiet: writing `category: 'electronics'` rows and then `category: 5` rows left `where { category: 5 }` returning nothing, while the same rows in a numbers-only corpus answered correctly. The column store now keeps one posting column per (field, kind), where a kind is a JavaScript typeof class. The first kind a field sees keeps the historical `_column_index//` layout, so a single-kind field is byte-identical to what earlier versions wrote and an index written before this opens unchanged; each later kind takes its own column at `_column_index//k//`. Equality reads the column matching the query value's own kind, so `{c: 5}` and `{c: '5'}` match different rows and neither is coerced into the other. Ranges route by the kind of their bounds, and an unbounded range — the "has any value" probe behind `exists` — reads every kind. A mixed field orders by kind first, then by value, because a number and a string have no order between them. A value that cannot be encoded for the column its own kind selected now raises instead of being skipped: that path is unreachable by construction, and if it is ever reached it is the silent drop this change exists to end. Two neighbours fell out of the same routing. A boolean query value is now encoded to the 1/0 the column stores, so boolean equality matches at all. And an integer column widens to f64 the first time a non-integer arrives, so 4.5 is stored as itself rather than rounded to 5 and answering the wrong query. Field type inference reports every kind a field holds beside its dominant reading, rather than leaving callers to treat one type as the whole answer. Pins: mixed-kind equality in both write orders, `5` vs `'5'`, booleans mixed in, a numeric range over a mixed field's numbers, close/reopen keeping every typed posting, and an index in the pre-existing on-disk shape still reading. `tests/critical-neural-validation.test.ts` — which writes `category` as strings in one test and as numbers in another against one shared brain — passes whole for the first time. (cherry picked from commit a128f0eda5b450ebf9caeae8e78ecaceff04feeb) --- .../architecture/data-storage-architecture.md | 34 ++ src/indexes/columnStore/ColumnStore.ts | 578 ++++++++++++++---- src/indexes/columnStore/ColumnTailBuffer.ts | 40 +- src/indexes/columnStore/types.ts | 60 ++ src/utils/fieldTypeInference.ts | 83 ++- .../metadata-field-typing.unit.test.ts | 122 ++++ .../column-store-mixed-kind.test.ts | 241 ++++++++ 7 files changed, 1025 insertions(+), 133 deletions(-) create mode 100644 tests/regression/metadata-field-typing.unit.test.ts create mode 100644 tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts diff --git a/docs/architecture/data-storage-architecture.md b/docs/architecture/data-storage-architecture.md index 83b9e23a..12398747 100644 --- a/docs/architecture/data-storage-architecture.md +++ b/docs/architecture/data-storage-architecture.md @@ -217,6 +217,40 @@ membership queries at scale: `__words__` for tokenized text…). - `_blobs/_column_index/{field}/L0-NNNNNN.bin` — the actual level-0 run segments, stored through the shared `_blobs/.bin` binary convention. +- `_column_index/{field}/k/{kind}/…` — the same two files again, for a + **second value kind** on the same field (see below). Absent for a field that + holds one kind, which is nearly all of them. + +### One posting column per (field, kind) + +A field is not obliged to hold one type of value. `category` may carry +`'electronics'` on some rows and `5` on others, and both are real values of +that field. A segment, though, has one encoding — i64, f64, UTF-8, or boolean +— so a field that holds several kinds gets **one column per kind**: + +- The first kind a field ever sees owns the plain `_column_index/{field}/` + layout above. A single-kind field is therefore byte-identical to what earlier + versions wrote, and an index written before typed postings opens unchanged. +- Every later kind gets its own column beside it at + `_column_index/{field}/k/{kind}/`, where `{kind}` is `number`, `string` or + `boolean`. + +What that buys at query time: + +| | | +|---|---| +| **Equality** | Answered from the column matching the **query value's own kind**. `where {category: 5}` reads the number postings; `where {category: '5'}` reads the string postings. Neither borrows the other's rows — a row written with the number `5` is not a row whose category is the text `'5'`. | +| **A kind the field never held** | Matches nothing. That is the true answer, not a coerced one. | +| **Ranges** | Routed by the kind of the bounds: numeric bounds read the numeric postings and ignore the field's strings. An **unbounded** range is the "has any value here" probe behind `exists`, and reads every kind. | +| **`orderBy`** | A number and a string have no order between them, so a mixed field orders by kind first (number, string, boolean) and by value within a kind. A single-kind field sorts exactly as it always did. | +| **Numbers** | One kind, one column: an integer column is written as i64 and widens to f64 the first time a non-integer arrives, so `4.5` is stored as itself rather than rounded. | + +`null` and `undefined` are not kinds and are never posted; their absence is +what the `exists` / `missing` operators read. + +Older readers are unaffected by the additional columns: they see the field's +primary column exactly where it has always been, and a `k/{kind}` directory is +simply a name they never query. Sparse per-field indexes, roaring-bitmap chunks, and zone-map/bloom segments additionally live as bucketed keys under `_system/idx/` (see §3). Which path diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 48f4a963..6bff86d4 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -23,7 +23,10 @@ import type { ColumnStoreProvider, SegmentMeta } from './types.js' import { ValueType, DEFAULT_FLUSH_THRESHOLD, - FLAG_MULTI_VALUE + FLAG_MULTI_VALUE, + POSTING_KINDS, + KIND_PATH_SEGMENT, + type PostingKind } from './types.js' import { ColumnTailBuffer } from './ColumnTailBuffer.js' import { ColumnManifest } from './ColumnManifest.js' @@ -52,10 +55,89 @@ interface HeapEntry { value: number | string entityIntId: number cursorIndex: number + /** + * Rank of the posting kind this entry came from, from {@link POSTING_KINDS}. + * A mixed-kind field has no natural total order, so the merge orders by kind + * first and by value within a kind. + */ + kindRank: number /** Iterator for the cursor — call next() to advance */ iterator: Generator } +/** + * One physical posting column: a (field, kind) pair and the key every internal + * map and every storage path uses for it. + */ +interface KindColumn { + /** The field as the query language names it. */ + field: string + /** The kind of value this column holds. */ + kind: PostingKind + /** + * Internal map / storage key. The field's PRIMARY kind uses the bare field + * name — the historical layout — and every other kind uses + * `//`. + */ + key: string +} + +/** + * The KIND a value indexes under — its JavaScript `typeof` class, not its + * storage encoding. + * + * Anything that is not a number, string or boolean indexes as a string, which + * is the `String(value)` treatment those values already received. `null` and + * `undefined` never reach here: `addEntity` skips them, and their absence is + * what the `exists` / `missing` operators read. + * + * @param value - The value about to be indexed or queried + * @returns The posting kind that owns this value + */ +function kindOfValue(value: unknown): PostingKind { + const t = typeof value + if (t === 'number') return 'number' + if (t === 'boolean') return 'boolean' + return 'string' +} + +/** + * The segment encoding a fresh column of this kind starts with. + * + * Only the number kind has a choice: an integer column starts as i64 and + * widens to f64 the first time a non-integer arrives + * ({@link ColumnTailBuffer.promoteToFloat}). + */ +function initialValueTypeFor(kind: PostingKind, firstValue: unknown): ValueType { + switch (kind) { + case 'boolean': + return ValueType.Boolean + case 'string': + return ValueType.String + case 'number': + return Number.isInteger(firstValue) ? ValueType.Number : ValueType.Float + } +} + +/** + * The kind a column of this encoding holds — the inverse of + * {@link initialValueTypeFor}, used to read a kind back off a manifest written + * before typed postings existed. + */ +function kindOfValueType(valueType: ValueType): PostingKind { + switch (valueType) { + case ValueType.Boolean: + return 'boolean' + case ValueType.String: + return 'string' + case ValueType.Number: + case ValueType.Float: + return 'number' + default: + throw new Error(`Unknown ValueType: ${valueType}`) + } +} + /** * Unified column store coordinator. * @@ -121,9 +203,19 @@ export class ColumnStore implements ColumnStoreProvider { */ private deletedEntities: Map = new Map() - /** Known field value types (inferred from first write). */ + /** Segment encoding per COLUMN key (not per field — a field has one per kind). */ private fieldTypes: Map = new Map() + /** + * Every posting column a field owns: field → kind → column key. + * + * This is the map that ends the first-writer type freeze. A field's first + * kind takes the bare field name as its column key, keeping the historical + * on-disk layout; each later kind takes its own column beside it. Nothing is + * coerced across kinds and nothing is dropped for being the wrong type. + */ + private fieldColumns: Map> = new Map() + /** Whether init() has completed. */ private initialized = false @@ -140,6 +232,128 @@ export class ColumnStore implements ColumnStoreProvider { this.l0CompactionTrigger = config?.l0CompactionTrigger ?? 4 } + // ========================================================================= + // Posting columns: (field, kind) → one physical column + // ========================================================================= + + /** + * Storage / map key for a (field, kind) column. + * + * `primary` is the kind that owns the bare field name. It is whichever kind + * the field saw first, which for an index written before typed postings is + * simply the kind of its single manifest — so the historical layout is + * preserved rather than migrated. + */ + private static columnKeyFor(field: string, kind: PostingKind, primary: PostingKind | null): string { + return primary === null || kind === primary + ? field + : `${field}/${KIND_PATH_SEGMENT}/${kind}` + } + + /** + * Split a discovered manifest path back into its (field, kind) column, or + * `null` when the path names a field's primary column rather than a kind + * column. `/k/` is the only shape that reads as a kind column, + * and only for a `` this version knows. + */ + private static parseKindColumnKey(key: string): { field: string; kind: PostingKind } | null { + const marker = `/${KIND_PATH_SEGMENT}/` + const at = key.lastIndexOf(marker) + if (at <= 0) return null + const kind = key.slice(at + marker.length) + if (!POSTING_KINDS.includes(kind as PostingKind)) return null + return { field: key.slice(0, at), kind: kind as PostingKind } + } + + /** Record a discovered or freshly created column against its field. */ + private registerColumn(field: string, kind: PostingKind, key: string): void { + let byKind = this.fieldColumns.get(field) + if (!byKind) { + byKind = new Map() + this.fieldColumns.set(field, byKind) + } + const existing = byKind.get(kind) + if (existing !== undefined && existing !== key) { + // Two columns claiming one (field, kind) means the layout on disk is not + // one this writer could have produced. Serving it would silently answer + // from half the postings, so say which two and stop. + throw new Error( + `ColumnStore: field '${field}' has two '${kind}' posting columns on ` + + `disk ('${existing}' and '${key}'). The column index layout is ` + + `inconsistent — rebuild/repair the metadata index rather than ` + + `serving from one half of it.` + ) + } + byKind.set(kind, key) + } + + /** The column key for this (field, kind), or `null` if the field has no such kind. */ + private columnKey(field: string, kind: PostingKind): string | null { + return this.fieldColumns.get(field)?.get(kind) ?? null + } + + /** + * The column key for this (field, kind), creating the registration if the + * field has not seen this kind before. Write path only. + */ + private ensureColumnKey(field: string, kind: PostingKind): string { + const byKind = this.fieldColumns.get(field) + const existing = byKind?.get(kind) + if (existing !== undefined) return existing + + // The primary kind is the one already holding the bare field name, if any. + let primary: PostingKind | null = null + if (byKind) { + for (const [k, key] of byKind) { + if (key === field) { primary = k; break } + } + } + const key = ColumnStore.columnKeyFor(field, kind, primary) + this.registerColumn(field, kind, key) + return key + } + + /** + * Every posting column this field owns, in {@link POSTING_KINDS} order. + * + * Read doors that are not about one particular value — an unbounded range + * used as an "any value present" probe, distinct values, sorting — fan out + * over all of them. + */ + private columnsForField(field: string): KindColumn[] { + const byKind = this.fieldColumns.get(field) + if (!byKind) return [] + const out: KindColumn[] = [] + for (const kind of POSTING_KINDS) { + const key = byKind.get(kind) + if (key !== undefined) out.push({ field, kind, key }) + } + return out + } + + /** + * Which value kinds this field actually holds, in {@link POSTING_KINDS} + * order — the honest answer to "what type is this field?". + * + * A field that carries both `'electronics'` and `5` reports + * `['number', 'string']`, not whichever of them was written first. + * + * @param field - Field name + * @returns Every kind with at least one posting, or `[]` for an unknown field + */ + getFieldKinds(field: string): PostingKind[] { + return this.columnsForField(field) + .filter((c) => this.columnHasData(c.key)) + .map((c) => c.kind) + } + + /** Does this physical column hold any postings (persisted or buffered)? */ + private columnHasData(key: string): boolean { + const manifest = this.manifests.get(key) + const buffer = this.tailBuffers.get(key) + return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0) + } + /** * Initialize the column store: discover existing field manifests. */ @@ -157,11 +371,23 @@ export class ColumnStore implements ColumnStoreProvider { }).listObjectsUnderPath(this.basePath + '/') for (const path of paths) { if (path.endsWith('/MANIFEST.json')) { - const fieldName = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '') - const manifest = new ColumnManifest(fieldName, this.basePath) + // The discovered name is a COLUMN key: either a bare field (that + // field's primary kind, which is every column an index written + // before typed postings has) or `/k/` for a second + // kind that arrived on a field later. + const columnKey = path.replace(this.basePath + '/', '').replace('/MANIFEST.json', '') + const manifest = new ColumnManifest(columnKey, this.basePath) await manifest.load(storage) - this.manifests.set(fieldName, manifest) - this.fieldTypes.set(fieldName, manifest.valueType) + this.manifests.set(columnKey, manifest) + this.fieldTypes.set(columnKey, manifest.valueType) + + const parsed = ColumnStore.parseKindColumnKey(columnKey) + if (parsed) { + this.registerColumn(parsed.field, parsed.kind, columnKey) + } else { + this.registerColumn(columnKey, kindOfValueType(manifest.valueType), columnKey) + } + const fieldName = columnKey // Load global deleted bitmap if it exists. Raw blob preferred // (2.4.0 #4 cortex-shared format); legacy envelope fallback for @@ -264,26 +490,43 @@ export class ColumnStore implements ColumnStoreProvider { /** * Point filter: find entities where field equals value. * - * Searches all segments + tail buffer, returns union as roaring bitmap. - * Excludes globally deleted entities. + * The QUERY VALUE'S OWN KIND picks the posting column, and only that column + * is read. `where {category: 5}` answers from the number postings and + * `where {category: '5'}` from the string postings — neither borrows the + * other's rows, because a row written with the number `5` is not a row whose + * category is the text `'5'`. + * + * A field that has never seen this kind matches nothing, which is the true + * answer rather than a coerced one. + * + * Searches all segments + tail buffer of that column, returns the union as a + * roaring bitmap. Excludes globally deleted entities. */ async filter(field: string, value: unknown): Promise { const result = new RoaringBitmap32() - const deleted = this.deletedEntities.get(field) + const columnKey = this.columnKey(field, kindOfValue(value)) + if (columnKey === null) return result + + // The query value takes the column's encoding — a boolean queried against + // a boolean column has to become the 1/0 the column stores. + const encoded = this.normalizeValue(value, this.fieldTypes.get(columnKey) ?? ValueType.String) + if (encoded === undefined) return result + + const deleted = this.deletedEntities.get(columnKey) // Search segments - const cursors = await this.getSegmentCursors(field) + const cursors = await this.getSegmentCursors(columnKey) for (const cursor of cursors) { - const ids = cursor.getEntityIdsForValue(value as number | string) + const ids = cursor.getEntityIdsForValue(encoded) for (const id of ids) { if (!deleted || !deleted.has(id)) result.add(id) } } // Search tail buffer - const tailCursor = this.getTailBufferCursor(field) + const tailCursor = this.getTailBufferCursor(columnKey) if (tailCursor) { - const ids = tailCursor.getEntityIdsForValue(value as number | string) + const ids = tailCursor.getEntityIdsForValue(encoded) for (const id of ids) { if (!deleted || !deleted.has(id)) result.add(id) } @@ -324,22 +567,26 @@ export class ColumnStore implements ColumnStoreProvider { const out = new Map() if (wanted.size === 0 || !this.hasField(field)) return out - const deleted = this.deletedEntities.get(field) - const take = (entry: { value: number | string; entityIntId: number }): void => { - if (!wanted.has(entry.entityIntId)) return - if (deleted && deleted.has(entry.entityIntId)) return - out.set(entry.entityIntId, entry.value) - } + // Every kind the field holds is read, in POSTING_KINDS order — a value an + // entity wrote as a string is still that entity's value for this field. + for (const column of this.columnsForField(field)) { + const deleted = this.deletedEntities.get(column.key) + const take = (entry: { value: number | string; entityIntId: number }): void => { + if (!wanted.has(entry.entityIntId)) return + if (deleted && deleted.has(entry.entityIntId)) return + out.set(entry.entityIntId, entry.value) + } - // Segments oldest -> newest, then the tail: a later write overwrites an - // earlier one for the same id. - const cursors = await this.getSegmentCursors(field) - for (const cursor of cursors) { - for (const entry of cursor.iterateForward()) take(entry) - } - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) take(entry) + // Segments oldest -> newest, then the tail: a later write overwrites an + // earlier one for the same id. + const cursors = await this.getSegmentCursors(column.key) + for (const cursor of cursors) { + for (const entry of cursor.iterateForward()) take(entry) + } + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) take(entry) + } } return out } @@ -363,41 +610,59 @@ export class ColumnStore implements ColumnStoreProvider { includeMax: boolean = true ): Promise { const result = new RoaringBitmap32() - const cursors = await this.getSegmentCursors(field) const hasMin = min !== undefined && min !== null const hasMax = max !== undefined && max !== null - for (const cursor of cursors) { - const lo = hasMin ? min as number | string : cursor.minValue - const hi = hasMax ? max as number | string : cursor.maxValue - if (lo === undefined || hi === undefined) continue - // Exclusivity applies only to an explicitly provided bound. A bound taken - // from the segment's own min/max is a real stored value and must stay - // inclusive, or the segment's boundary entities would be wrongly dropped. - const ids = cursor.getEntityIdsInRange( - lo, - hi, - hasMin ? includeMin : true, - hasMax ? includeMax : true - ) - for (const id of ids) result.add(id) - } + // The BOUNDS pick the column: numeric bounds read the numeric postings, + // string bounds the string postings. An unbounded call is not a range at + // all — it is the "has any value here" probe behind `exists` — so it fans + // out over every kind the field holds. + const columns: KindColumn[] = hasMin + ? this.columnsForKind(field, kindOfValue(min)) + : hasMax + ? this.columnsForKind(field, kindOfValue(max)) + : this.columnsForField(field) - // Tail buffer range: linear scan (tail is small) - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) { - const v = entry.value as any - const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any)) - const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any)) - if (loOk && hiOk) result.add(entry.entityIntId) + for (const column of columns) { + const cursors = await this.getSegmentCursors(column.key) + for (const cursor of cursors) { + const lo = hasMin ? min as number | string : cursor.minValue + const hi = hasMax ? max as number | string : cursor.maxValue + if (lo === undefined || hi === undefined) continue + // Exclusivity applies only to an explicitly provided bound. A bound taken + // from the segment's own min/max is a real stored value and must stay + // inclusive, or the segment's boundary entities would be wrongly dropped. + const ids = cursor.getEntityIdsInRange( + lo, + hi, + hasMin ? includeMin : true, + hasMax ? includeMax : true + ) + for (const id of ids) result.add(id) + } + + // Tail buffer range: linear scan (tail is small) + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) { + const v = entry.value as any + const loOk = !hasMin || (includeMin ? v >= (min as any) : v > (min as any)) + const hiOk = !hasMax || (includeMax ? v <= (max as any) : v < (max as any)) + if (loOk && hiOk) result.add(entry.entityIntId) + } } } return result } + /** The single column for this (field, kind), as a list, or empty if absent. */ + private columnsForKind(field: string, kind: PostingKind): KindColumn[] { + const key = this.columnKey(field, kind) + return key === null ? [] : [{ field, kind, key }] + } + /** * Sort top-K: return K entity int IDs in sorted order (u64-safe BigInt). * @@ -428,18 +693,21 @@ export class ColumnStore implements ColumnStoreProvider { */ async getFilterValues(field: string): Promise { const valueSet = new Set() - const cursors = await this.getSegmentCursors(field) - for (const cursor of cursors) { - for (const entry of cursor.iterateForward()) { - valueSet.add(String(entry.value)) + for (const column of this.columnsForField(field)) { + const cursors = await this.getSegmentCursors(column.key) + + for (const cursor of cursors) { + for (const entry of cursor.iterateForward()) { + valueSet.add(String(entry.value)) + } } - } - const tailCursor = this.getTailBufferCursor(field) - if (tailCursor) { - for (const entry of tailCursor.iterateForward()) { - valueSet.add(String(entry.value)) + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + for (const entry of tailCursor.iterateForward()) { + valueSet.add(String(entry.value)) + } } } @@ -450,9 +718,7 @@ export class ColumnStore implements ColumnStoreProvider { * Check if a field has any indexed data. */ hasField(field: string): boolean { - const manifest = this.manifests.get(field) - const buffer = this.tailBuffers.get(field) - return (manifest !== undefined && !manifest.isEmpty()) || (buffer !== undefined && buffer.size > 0) + return this.columnsForField(field).some((c) => this.columnHasData(c.key)) } /** @@ -462,12 +728,11 @@ export class ColumnStore implements ColumnStoreProvider { * store will actually serve queries from. */ getIndexedFields(): string[] { + // Names FIELDS, not columns: a field carrying two kinds is one name here, + // the same name a caller queries with. const fields = new Set() - for (const [field, manifest] of this.manifests) { - if (!manifest.isEmpty()) fields.add(field) - } - for (const [field, buffer] of this.tailBuffers) { - if (buffer.size > 0) fields.add(field) + for (const [field] of this.fieldColumns) { + if (this.hasField(field)) fields.add(field) } return Array.from(fields).sort() } @@ -482,12 +747,16 @@ export class ColumnStore implements ColumnStoreProvider { getFieldSizeSummary(): Array<{ field: string; segmentCount: number; tailSize: number }> { const summary: Array<{ field: string; segmentCount: number; tailSize: number }> = [] for (const field of this.getIndexedFields()) { - const manifest = this.manifests.get(field) - const buffer = this.tailBuffers.get(field) - const segmentCount = manifest && !manifest.isEmpty() - ? manifest.getAllSegments().length - : 0 - const tailSize = buffer ? buffer.size : 0 + // Summed across the field's kind columns — the caller asked about a + // field, and a field's size is all of the postings under its name. + let segmentCount = 0 + let tailSize = 0 + for (const column of this.columnsForField(field)) { + const manifest = this.manifests.get(column.key) + const buffer = this.tailBuffers.get(column.key) + if (manifest && !manifest.isEmpty()) segmentCount += manifest.getAllSegments().length + if (buffer) tailSize += buffer.size + } summary.push({ field, segmentCount, tailSize }) } return summary @@ -515,6 +784,8 @@ export class ColumnStore implements ColumnStoreProvider { this.segmentCache.clear() this.manifests.clear() this.deletedEntities.clear() + this.fieldColumns.clear() + this.fieldTypes.clear() this.initialized = false } @@ -523,32 +794,64 @@ export class ColumnStore implements ColumnStoreProvider { // ========================================================================= /** - * Push a single value to a field's tail buffer. - * Creates the buffer and manifest if first write to this field. - * Infers ValueType from the first value seen. + * Push a single value to the posting column for its (field, KIND). + * + * The value's own kind picks the column — a string goes to the field's + * string postings, a number to its number postings — so a field carrying + * `'electronics'` and `5` keeps both, each answerable by an equality filter + * of its own kind. Under the first-writer type freeze this method replaced, + * the first value's type became the field's type and every later value of + * another kind was coerced to it or, when coercion failed, dropped with no + * error at all. + * + * Creates the column's buffer and manifest on its first value. */ private pushToBuffer(field: string, value: unknown, entityIntId: number, isMultiValue: boolean): void { - let buffer = this.tailBuffers.get(field) + const kind = kindOfValue(value) + const columnKey = this.ensureColumnKey(field, kind) + + let buffer = this.tailBuffers.get(columnKey) if (!buffer) { - const valueType = this.inferValueType(value) - buffer = new ColumnTailBuffer(field, valueType, this.flushThreshold) - this.tailBuffers.set(field, buffer) - this.fieldTypes.set(field, valueType) + // A reopened column takes its encoding from its manifest — an integer + // column that widened to f64 in an earlier session stays widened. + const valueType = + this.manifests.get(columnKey)?.valueType ?? initialValueTypeFor(kind, value) + buffer = new ColumnTailBuffer(columnKey, valueType, this.flushThreshold) + this.tailBuffers.set(columnKey, buffer) + this.fieldTypes.set(columnKey, valueType) // Ensure manifest exists - if (!this.manifests.has(field)) { - const manifest = new ColumnManifest(field, this.basePath) + if (!this.manifests.has(columnKey)) { + const manifest = new ColumnManifest(columnKey, this.basePath) manifest.valueType = valueType manifest.multiValue = isMultiValue - this.manifests.set(field, manifest) + this.manifests.set(columnKey, manifest) } } - // Normalize value to the column type - const normalizedValue = this.normalizeValue(value, buffer.valueType) - if (normalizedValue !== undefined) { - buffer.add(normalizedValue, entityIntId) + // An integer column widens the first time a non-integer number arrives, so + // the value is stored as itself instead of rounded to the nearest integer. + if (kind === 'number' && buffer.valueType === ValueType.Number && !Number.isInteger(value)) { + buffer.promoteToFloat() + this.fieldTypes.set(columnKey, ValueType.Float) + const manifest = this.manifests.get(columnKey) + if (manifest) manifest.valueType = ValueType.Float } + + const normalizedValue = this.normalizeValue(value, buffer.valueType) + if (normalizedValue === undefined) { + // Unreachable by construction: the column was chosen BY this value's + // kind, so the encoding always accepts it. Reaching here would mean a + // value had been silently dropped from the index — the exact failure + // typed postings exist to end — so it is an error, never a skip. + throw new Error( + `ColumnStore: field '${field}' rejected a ${kind} value for its own ` + + `${ValueType[buffer.valueType]} posting column. The value would have ` + + `been dropped from the index while the row stayed readable by id — ` + + `this is a kind-routing bug, not a value the caller may ignore.` + ) + } + buffer.add(normalizedValue, entityIntId) } /** @@ -677,8 +980,15 @@ export class ColumnStore implements ColumnStoreProvider { /** Torn-segment quarantine entries for a field (observability + heal input). */ quarantinedSegments(field: string): Array<{ segment: string; error: string; hits: number }> { const out: Array<{ segment: string; error: string; hits: number }> = [] - for (const [key, q] of this.segmentQuarantine) { - if (key.startsWith(`${field}:`)) out.push({ segment: key.slice(field.length + 1), error: q.error, hits: q.hits }) + // Across every kind column of the field — a torn segment in the string + // postings is this field's torn segment as much as one in the numbers. + for (const column of this.columnsForField(field)) { + const prefix = `${column.key}:` + for (const [key, q] of this.segmentQuarantine) { + if (key.startsWith(prefix)) { + out.push({ segment: key.slice(prefix.length), error: q.error, hits: q.hits }) + } + } } return out } @@ -850,17 +1160,22 @@ export class ColumnStore implements ColumnStoreProvider { k: number, filterBitmap: RoaringBitmap32 | null ): Promise { - // Collect all cursors (segments + tail buffer) - const segCursors = await this.getSegmentCursors(field) - const tailCursor = this.getTailBufferCursor(field) - - // Create iterators for each cursor in the specified direction + // Collect cursors across EVERY kind the field holds. A single-kind field — + // nearly all of them — merges exactly the cursors it always did. const iterators: Generator[] = [] - for (const cursor of segCursors) { - iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward()) - } - if (tailCursor) { - iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward()) + const iteratorKindRank: number[] = [] + for (const column of this.columnsForField(field)) { + const kindRank = POSTING_KINDS.indexOf(column.kind) + const segCursors = await this.getSegmentCursors(column.key) + for (const cursor of segCursors) { + iterators.push(order === 'asc' ? cursor.iterateForward() : cursor.iterateBackward()) + iteratorKindRank.push(kindRank) + } + const tailCursor = this.getTailBufferCursor(column.key) + if (tailCursor) { + iterators.push(order === 'asc' ? tailCursor.iterateForward() : tailCursor.iterateBackward()) + iteratorKindRank.push(kindRank) + } } if (iterators.length === 0) return [] @@ -874,16 +1189,21 @@ export class ColumnStore implements ColumnStoreProvider { value: next.value.value, entityIntId: next.value.entityIntId, cursorIndex: i, + kindRank: iteratorKindRank[i], iterator: iterators[i] }) } } - // Heapify - const isString = (this.fieldTypes.get(field) ?? ValueType.Number) === ValueType.String + // Heapify. A number and a string have no ordering between them, so a + // mixed-kind field orders by KIND first (POSTING_KINDS order) and by value + // within a kind — one defined total order instead of a comparison whose + // answer depends on which value happened to be on the left. const compare = (a: HeapEntry, b: HeapEntry): number => { let cmp: number - if (isString) { + if (a.kindRank !== b.kindRank) { + cmp = a.kindRank - b.kindRank + } else if (POSTING_KINDS[a.kindRank] === 'string') { cmp = compareCodePoints(String(a.value), String(b.value)) } else { cmp = (a.value as number) - (b.value as number) @@ -915,6 +1235,7 @@ export class ColumnStore implements ColumnStoreProvider { value: next.value.value, entityIntId: next.value.entityIntId, cursorIndex: top.cursorIndex, + kindRank: top.kindRank, iterator: top.iterator } } @@ -922,8 +1243,11 @@ export class ColumnStore implements ColumnStoreProvider { this.heapDown(heap, 0, compare) } - // Apply global deleted check, filter, and dedup - const deleted = this.deletedEntities.get(field) + // Apply global deleted check, filter, and dedup. The deleted bitmap is + // per COLUMN, and the entry came from the column its kind names. + const deleted = this.deletedEntities.get( + this.columnKey(field, POSTING_KINDS[top.kindRank]) ?? field + ) if (deleted && deleted.has(top.entityIntId)) continue if (seen.has(top.entityIntId)) continue if (filterBitmap && !filterBitmap.has(top.entityIntId)) continue @@ -965,35 +1289,31 @@ export class ColumnStore implements ColumnStoreProvider { } /** - * Infer ValueType from a JavaScript value. - */ - private inferValueType(value: unknown): ValueType { - if (typeof value === 'boolean') return ValueType.Boolean - if (typeof value === 'number') { - return Number.isInteger(value) ? ValueType.Number : ValueType.Float - } - return ValueType.String - } - - /** - * Normalize a JavaScript value to the column's ValueType. + * Encode a value for the column its own kind selected. + * + * This does NOT convert between kinds. It used to: a string reaching a + * numeric column was run through `Number(value)`, and a number reaching a + * numeric column was run through `Math.round`, so `'electronics'` became + * `NaN` and vanished while `4.5` became `5` and answered the wrong query. + * Kind routing removes the need for either — the only work left is picking + * the encoding the column already committed to. + * + * @returns The encoded value, or `undefined` if the value does not belong in + * this column at all — which the caller treats as a routing bug and + * raises, never as a value to skip. */ private normalizeValue(value: unknown, type: ValueType): number | string | undefined { switch (type) { case ValueType.Number: - if (typeof value === 'number') return Math.round(value) - if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : Math.round(n) } - if (typeof value === 'boolean') return value ? 1 : 0 - return undefined + // Integer column. Non-integers widen it to Float before reaching here. + return typeof value === 'number' && Number.isInteger(value) ? value : undefined case ValueType.Float: - if (typeof value === 'number') return value - if (typeof value === 'string') { const n = Number(value); return isNaN(n) ? undefined : n } - return undefined + return typeof value === 'number' ? value : undefined case ValueType.Boolean: - if (typeof value === 'boolean') return value ? 1 : 0 - if (typeof value === 'number') return value ? 1 : 0 - return undefined + return typeof value === 'boolean' ? (value ? 1 : 0) : undefined case ValueType.String: + // The string kind is also where objects and bigints land, exactly as + // they always did. return String(value) default: return undefined diff --git a/src/indexes/columnStore/ColumnTailBuffer.ts b/src/indexes/columnStore/ColumnTailBuffer.ts index c5874ac2..e730f884 100644 --- a/src/indexes/columnStore/ColumnTailBuffer.ts +++ b/src/indexes/columnStore/ColumnTailBuffer.ts @@ -55,8 +55,12 @@ export class ColumnTailBuffer { /** Field name this buffer is for. */ readonly fieldName: string - /** Value type determines sort comparator. */ - readonly valueType: ValueType + /** + * Value type determines sort comparator and segment encoding. + * + * Widened in place by {@link promoteToFloat} — never otherwise reassigned. + */ + valueType: ValueType /** Flush threshold. */ readonly threshold: number @@ -81,6 +85,38 @@ export class ColumnTailBuffer { this.threshold = threshold } + /** + * Widen an integer column to floating point, losslessly and in place. + * + * The number posting kind holds every JavaScript number, but a segment picks + * ONE encoding: i64 for integers, f64 for the rest. A column that has only + * ever seen integers is written as i64; the first non-integer to arrive + * widens it here, so that value is stored as itself instead of being rounded + * to the nearest integer with no error — the rounding that made `4.5` and + * `5.5` both answer `where {score: 5}` and neither answer its own value. + * + * Widening is lossless in both directions it has to be: every value already + * buffered is an integer, and every integer is exactly representable as f64. + * Segments already on disk keep their own i64 encoding in their own headers + * and keep decoding by it — only segments written from here on are f64. + * + * @throws Error if called on a column that is not an integer column — the + * only legal widening is Number → Float, and any other request is a bug in + * the caller's kind routing rather than something to absorb quietly. + */ + promoteToFloat(): void { + if (this.valueType === ValueType.Float) return + if (this.valueType !== ValueType.Number) { + throw new Error( + `ColumnTailBuffer '${this.fieldName}': cannot widen a ` + + `${ValueType[this.valueType]} column to Float — only an integer ` + + `(Number) column widens, and this call means a value reached the ` + + `wrong kind's column` + ) + } + this.valueType = ValueType.Float + } + /** * Add a (value, entityIntId) entry to the buffer. * diff --git a/src/indexes/columnStore/types.ts b/src/indexes/columnStore/types.ts index 71dd99a0..ee949bd0 100644 --- a/src/indexes/columnStore/types.ts +++ b/src/indexes/columnStore/types.ts @@ -58,6 +58,53 @@ export enum ValueType { Boolean = 3 } +/** + * The KIND of a value, as the query language sees it. + * + * A kind is a JavaScript `typeof` class, not a storage encoding: `5` and `5.5` + * are one kind (`'number'`) held in one posting column, even though they need + * different segment encodings (i64 vs f64 — see {@link ValueType}). + * + * A field holds ONE POSTING COLUMN PER KIND, so `category` may carry string + * values and number values at the same time and answer equality on each. This + * replaces the first-writer type freeze, under which the first value's type + * became the field's type and every later value of another kind was coerced — + * or, when coercion failed (`Number('electronics')`), dropped from the index + * with no error: the row stayed readable by id and by vector but vanished from + * every equality filter on that field. + * + * Kinds do not coerce into one another at query time either: `where {c: 5}` + * matches rows written with the NUMBER `5`, and `where {c: '5'}` matches rows + * written with the STRING `'5'`. Neither ever matches the other. + * + * Values that are none of these three (objects, bigints) index as strings — + * the same `String(value)` treatment they received before. + */ +export type PostingKind = 'number' | 'string' | 'boolean' + +/** + * Every posting kind, in the order that defines cross-kind sort position. + * + * A mixed-kind field has no natural total order — a number does not compare + * with a string — so `sortTopK` orders by KIND first (numbers, then strings, + * then booleans) and by value within a kind. A single-kind field, which is + * nearly every field, sorts exactly as it always did. + */ +export const POSTING_KINDS: readonly PostingKind[] = ['number', 'string', 'boolean'] + +/** + * Path segment marking a field's NON-PRIMARY kind columns on disk. + * + * The first kind a field ever sees keeps the historical layout — + * `//MANIFEST.json` and `//L0-NNNNNN` — so every + * index written before typed postings opens unchanged, and the byte-for-byte + * interchange with the native column store is untouched for the single-kind + * fields that are nearly all of them. A second kind arriving on the same field + * gets its own column at `//k//…` rather than overwriting or + * being coerced into the first. + */ +export const KIND_PATH_SEGMENT = 'k' + // --------------------------------------------------------------------------- // Segment header and footer // --------------------------------------------------------------------------- @@ -267,6 +314,19 @@ export interface ColumnStoreProvider { */ hasField(field: string): boolean + /** + * Which value KINDS this field actually holds, in {@link POSTING_KINDS} + * order — the honest answer to "what type is this field?" for a field that + * carries more than one. + * + * OPTIONAL so an implementation written against the pre-typed-postings + * contract still satisfies this interface; feature-detect before calling. + * + * @param field - Field name + * @returns Every kind with at least one posting, or `[]` for an unknown field + */ + getFieldKinds?(field: string): PostingKind[] + /** * Flush all in-memory tail buffers to L0 segments on disk. * Saves all manifests. diff --git a/src/utils/fieldTypeInference.ts b/src/utils/fieldTypeInference.ts index 36a415b2..0f085f8c 100644 --- a/src/utils/fieldTypeInference.ts +++ b/src/utils/fieldTypeInference.ts @@ -55,8 +55,30 @@ export enum FieldType { */ export interface FieldTypeInfo { field: string + /** + * The DOMINANT reading of the field — one type, the most specific one every + * sampled value satisfies. + * + * A field is not obliged to hold one kind, so this is not the whole answer + * for a field that holds several. Read {@link kinds} beside it: a field + * carrying `'electronics'` and `5` infers as STRING here and reports + * `['number', 'string']` there, and the metadata index keeps a separate + * posting column for each of them. + */ inferredType: FieldType confidence: number // 0-1 confidence score + /** + * Every value KIND observed in the sample, in the order + * number → string → boolean. More than one entry means a genuinely + * mixed field, and every one of those kinds is independently filterable. + * + * Kinds are JavaScript `typeof` classes, one level coarser than + * {@link FieldType}: a UUID and a category name are both `'string'`, and an + * integer and a timestamp are both `'number'`. + * + * Optional only for cached analyses written before this was reported. + */ + kinds?: Array<'number' | 'string' | 'boolean'> sampleSize: number // Number of values analyzed lastUpdated: number // Timestamp of last analysis detectionMethod: 'value' // Always 'value' (no fallbacks!) @@ -133,14 +155,71 @@ export class FieldTypeInference { } /** - * Analyze values to determine field type + * Analyze values to determine field type, and report every KIND the field + * actually holds alongside it. + * + * The classification below picks ONE type, because every one of its + * heuristics asks `samples.every(...)`: a field carrying `'electronics'` and + * `5` satisfies none of them and lands on STRING. That single answer is true + * as far as it goes — string is the dominant reading — but on its own it + * says nothing about the numbers also in the field, and a caller that treats + * it as the field's only type reproduces the first-writer freeze the index + * itself no longer has. {@link FieldTypeInfo.kinds} carries the rest. + */ + private async analyzeValues(field: string, values: any[]): Promise { + const info = await this.classifyValues(field, values) + info.kinds = FieldTypeInference.observedKinds(values) + if (info.kinds.length > 1 && info.metadata) { + info.metadata.format = `${info.metadata.format} (field also holds: ${info.kinds + .filter((k) => k !== FieldTypeInference.kindOfType(info.inferredType)) + .join(', ')})` + } + return info + } + + /** + * The distinct value kinds present in a sample, in a stable order. + * + * Kinds are JavaScript `typeof` classes — the same classes the metadata + * index keeps separate posting columns for — not the finer + * {@link FieldType} readings, which are interpretations layered on top of + * them (a UUID and a category name are both the `string` kind). + */ + private static observedKinds(values: any[]): Array<'number' | 'string' | 'boolean'> { + const order: Array<'number' | 'string' | 'boolean'> = ['number', 'string', 'boolean'] + const seen = new Set<'number' | 'string' | 'boolean'>() + for (const v of values) { + if (v === null || v === undefined) continue + const t = typeof v + seen.add(t === 'number' ? 'number' : t === 'boolean' ? 'boolean' : 'string') + } + return order.filter((k) => seen.has(k)) + } + + /** The value kind a {@link FieldType} reading is an interpretation of. */ + private static kindOfType(type: FieldType): 'number' | 'string' | 'boolean' { + switch (type) { + case FieldType.BOOLEAN: + return 'boolean' + case FieldType.INTEGER: + case FieldType.FLOAT: + case FieldType.TIMESTAMP_MS: + case FieldType.TIMESTAMP_S: + return 'number' + default: + return 'string' + } + } + + /** + * Classify values into a single field type. * * Uses DuckDB-inspired type detection order: * BOOLEAN → INTEGER → FLOAT → DATE → TIMESTAMP → UUID → STRING * * No fallbacks - pure value-based detection */ - private async analyzeValues(field: string, values: any[]): Promise { + private async classifyValues(field: string, values: any[]): Promise { // Filter null/undefined values const validValues = values.filter(v => v !== null && v !== undefined) diff --git a/tests/regression/metadata-field-typing.unit.test.ts b/tests/regression/metadata-field-typing.unit.test.ts new file mode 100644 index 00000000..910d4f2a --- /dev/null +++ b/tests/regression/metadata-field-typing.unit.test.ts @@ -0,0 +1,122 @@ +/** + * @module metadata-field-typing.unit.test + * @description Regression: a metadata field that holds more than one value + * KIND stays fully filterable on every kind it holds. + * + * The defect this pins, reproduced on the released engine: the metadata index + * fixed a field's value type from the FIRST value it saw, and every later value + * of a different type was coerced to that type or, when coercion failed, + * dropped from the index in silence. Writing `category: 'electronics'` rows and + * then `category: 5` rows left `find({ where: { category: 5 } })` returning + * nothing — while the same rows in a numbers-only corpus answered correctly. + * The rows themselves were never lost: they stayed readable by id and by vector + * search, and only ever went missing from equality filters on that one field, + * which is what made it so quiet. + * + * Order is the whole point of these cases. Neither writer owns the field, so + * strings-then-numbers and numbers-then-strings must give the same answers. + */ + +import { describe, it, expect } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +/** A brain over memory storage, with a corpus written in the given order. */ +async function brainWith( + rows: Array<{ label: string; category: unknown }> +): Promise { + const brainy = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) + await brainy.init() + for (const row of rows) { + await brainy.add({ + data: `item ${row.label}`, + type: NounType.Thing, + metadata: { label: row.label, category: row.category } + }) + } + return brainy +} + +const labelsOf = (results: Array<{ metadata?: Record }>): string[] => + results.map((r) => String(r.metadata?.label)).sort() + +describe('regression: a mixed-kind metadata field filters on every kind', { timeout: 180_000 }, () => { + it('finds number rows written after string rows', async () => { + const brainy = await brainWith([ + { label: 'e1', category: 'electronics' }, + { label: 'f1', category: 'furniture' }, + { label: 'n1', category: 5 }, + { label: 'n2', category: 5 }, + { label: 'n3', category: 7 } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2']) + expect(labelsOf(await brainy.find({ where: { category: 7 }, limit: 100 }))).toEqual(['n3']) + expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1']) + expect(labelsOf(await brainy.find({ where: { category: 'furniture' }, limit: 100 }))).toEqual(['f1']) + } finally { + await brainy.close() + } + }) + + it('finds string rows written after number rows', async () => { + const brainy = await brainWith([ + { label: 'n1', category: 5 }, + { label: 'n2', category: 5 }, + { label: 'e1', category: 'electronics' }, + { label: 'e2', category: 'electronics' } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 'electronics' }, limit: 100 }))).toEqual(['e1', 'e2']) + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['n1', 'n2']) + } finally { + await brainy.close() + } + }) + + it('keeps `5` and `\'5\'` apart — a kind is part of the value, not a formatting detail', async () => { + const brainy = await brainWith([ + { label: 'num', category: 5 }, + { label: 'str', category: '5' } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: 5 }, limit: 100 }))).toEqual(['num']) + expect(labelsOf(await brainy.find({ where: { category: '5' }, limit: 100 }))).toEqual(['str']) + } finally { + await brainy.close() + } + }) + + it('serves booleans mixed into a field that already holds strings', async () => { + const brainy = await brainWith([ + { label: 's1', category: 'yes' }, + { label: 'b1', category: true }, + { label: 'b2', category: false } + ]) + try { + expect(labelsOf(await brainy.find({ where: { category: true }, limit: 100 }))).toEqual(['b1']) + expect(labelsOf(await brainy.find({ where: { category: false }, limit: 100 }))).toEqual(['b2']) + expect(labelsOf(await brainy.find({ where: { category: 'yes' }, limit: 100 }))).toEqual(['s1']) + } finally { + await brainy.close() + } + }) + + it('ranges over the numeric part of a mixed field', async () => { + const brainy = await brainWith([ + { label: 'unpriced', category: 'on request' }, + { label: 'cheap', category: 100 }, + { label: 'mid', category: 500 }, + { label: 'dear', category: 900 } + ]) + try { + const found = await brainy.find({ + where: { category: { greaterThan: 200 } }, + limit: 100 + }) + expect(labelsOf(found)).toEqual(['dear', 'mid']) + } finally { + await brainy.close() + } + }) +}) diff --git a/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts b/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts new file mode 100644 index 00000000..1ce21d1f --- /dev/null +++ b/tests/unit/indexes/columnStore/column-store-mixed-kind.test.ts @@ -0,0 +1,241 @@ +/** + * @module column-store-mixed-kind.test + * @description Typed posting lists: one field, several value KINDS, each + * answerable on its own. + * + * The behaviour these pin replaced a first-writer type freeze. The first value + * a field ever saw fixed that field's type; every later value of another kind + * was coerced to it, and when coercion failed — `Number('electronics')` — the + * value was dropped from the index with no error at all. The row stayed + * readable by id and by vector and vanished from every equality filter on the + * field. These tests therefore care about ORDER: strings-then-numbers and + * numbers-then-strings have to behave identically, because neither writer owns + * the field. + * + * Kinds never coerce into one another at query time either. `5` and `'5'` are + * different values and match different rows. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js' +import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' +import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' + +describe('ColumnStore — typed posting lists per (field, kind)', () => { + let storage: MemoryStorage + let idMapper: EntityIdMapper + let store: ColumnStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' }) + await idMapper.init() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + }) + + afterEach(async () => { + await store.close() + }) + + /** Resolve a filter to the sorted UUIDs it matched. */ + const uuidsOf = async (field: string, value: unknown): Promise => { + const bitmap = await store.filter(field, value) + return Array.from(bitmap) + .map((id) => idMapper.getUuid(Number(id))) + .filter((u): u is string => u !== undefined) + .sort() + } + + describe('equality answers on the query value’s own kind', () => { + it('serves numbers written AFTER strings on the same field', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'furniture' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n3')), { category: 7 }) + + // The numbers are in the index, though a string got there first. + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + expect(await uuidsOf('category', 7)).toEqual(['n3']) + // And the strings did not move. + expect(await uuidsOf('category', 'electronics')).toEqual(['s1']) + expect(await uuidsOf('category', 'furniture')).toEqual(['s2']) + }) + + it('serves strings written AFTER numbers on the same field', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + + // 'electronics' would have become NaN and been dropped under the freeze. + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + }) + + it('does not coerce a number query into the string postings, or back', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('num')), { code: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('str')), { code: '5' }) + + expect(await uuidsOf('code', 5)).toEqual(['num']) + expect(await uuidsOf('code', '5')).toEqual(['str']) + }) + + it('serves booleans mixed into a field that already holds strings and numbers', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { flag: 'yes' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { flag: 1 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { flag: true }) + store.addEntity(BigInt(idMapper.getOrAssign('b2')), { flag: false }) + + expect(await uuidsOf('flag', true)).toEqual(['b1']) + expect(await uuidsOf('flag', false)).toEqual(['b2']) + // `true` stores as 1 internally; that is an encoding, not a value. + expect(await uuidsOf('flag', 1)).toEqual(['n1']) + expect(await uuidsOf('flag', 'yes')).toEqual(['s1']) + }) + + it('answers nothing — not something coerced — for a kind the field never held', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + + expect(await uuidsOf('category', 5)).toEqual([]) + expect(await uuidsOf('category', true)).toEqual([]) + }) + + it('holds every kind across a flush, not just the one in the tail buffer', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + await store.flush() + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + }) + }) + + describe('range filters read the numeric postings', () => { + it('ranges over the numeric subset of a mixed field, ignoring its strings', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('cheap')), { price: 100 }) + store.addEntity(BigInt(idMapper.getOrAssign('mid')), { price: 500 }) + store.addEntity(BigInt(idMapper.getOrAssign('dear')), { price: 900 }) + store.addEntity(BigInt(idMapper.getOrAssign('unpriced')), { price: 'on request' }) + await store.flush() + + const inRange = await store.rangeQuery('price', 200, 1000) + const uuids = Array.from(inRange) + .map((id) => idMapper.getUuid(Number(id))) + .sort() + expect(uuids).toEqual(['dear', 'mid']) + }) + + it('an unbounded range still reports every kind — it is the “has a value” probe', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { mixed: 42 }) + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { mixed: 'text' }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { mixed: true }) + await store.flush() + + const anyValue = await store.rangeQuery('mixed') + const uuids = Array.from(anyValue) + .map((id) => idMapper.getUuid(Number(id))) + .sort() + expect(uuids).toEqual(['b1', 'n1', 's1']) + }) + }) + + describe('the index reports what a field actually holds', () => { + it('names every kind present, not the one that got there first', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + expect(store.getFieldKinds('category')).toEqual(['string']) + + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true }) + expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean']) + + // And the field is still ONE field by name. + expect(store.getIndexedFields()).toEqual(['category']) + expect(store.hasField('category')).toBe(true) + }) + + it('reports an unknown field as holding nothing', () => { + expect(store.getFieldKinds('never-written')).toEqual([]) + }) + }) + + describe('an integer column widens rather than rounding', () => { + it('keeps a non-integer written after integers as itself', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('a')), { score: 4 }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { score: 4.5 }) + store.addEntity(BigInt(idMapper.getOrAssign('c')), { score: 5 }) + await store.flush() + + // 4.5 used to round to 5 and answer `score === 5` alongside c. + expect(await uuidsOf('score', 4.5)).toEqual(['b']) + expect(await uuidsOf('score', 5)).toEqual(['c']) + expect(await uuidsOf('score', 4)).toEqual(['a']) + }) + }) + + describe('close then reopen', () => { + it('keeps every typed posting, on the same storage', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: true }) + store.addEntity(BigInt(idMapper.getOrAssign('f1')), { score: 1.5 }) + await store.flush() + await store.close() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + expect(store.getFieldKinds('category')).toEqual(['number', 'string', 'boolean']) + expect(await uuidsOf('category', 'electronics')).toEqual(['s1']) + expect(await uuidsOf('category', 5)).toEqual(['n1']) + expect(await uuidsOf('category', true)).toEqual(['b1']) + expect(await uuidsOf('score', 1.5)).toEqual(['f1']) + }) + + it('accepts new values of every kind after the reopen', async () => { + store.addEntity(BigInt(idMapper.getOrAssign('s1')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n1')), { category: 5 }) + await store.flush() + await store.close() + + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + store.addEntity(BigInt(idMapper.getOrAssign('s2')), { category: 'electronics' }) + store.addEntity(BigInt(idMapper.getOrAssign('n2')), { category: 5 }) + store.addEntity(BigInt(idMapper.getOrAssign('b1')), { category: false }) + await store.flush() + + expect(await uuidsOf('category', 'electronics')).toEqual(['s1', 's2']) + expect(await uuidsOf('category', 5)).toEqual(['n1', 'n2']) + expect(await uuidsOf('category', false)).toEqual(['b1']) + }) + + it('opens an index written by the pre-typed-postings shape and reads it unchanged', async () => { + // A single-kind field is byte-identical to what the old writer produced: + // one manifest at `_column_index//MANIFEST.json`, no kind + // subdirectory anywhere. That IS the old on-disk shape, so proving the + // new reader serves it proves an old index still opens. + store.addEntity(BigInt(idMapper.getOrAssign('a')), { status: 'active' }) + store.addEntity(BigInt(idMapper.getOrAssign('b')), { status: 'archived' }) + await store.flush() + + const keys = await (storage as unknown as { + listObjectsUnderPath: (prefix: string) => Promise + }).listObjectsUnderPath('_column_index/') + expect(keys.some((k) => k.includes('/k/'))).toBe(false) + + await store.close() + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + + expect(store.getFieldKinds('status')).toEqual(['string']) + expect(await uuidsOf('status', 'active')).toEqual(['a']) + }) + }) +}) From da7d2498bc8c2225d355437eeecfe4c0e5709899 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:12:20 -0700 Subject: [PATCH 146/150] =?UTF-8?q?docs(changelog):=20the=2010.4.12=20note?= =?UTF-8?q?,=20curated=20=E2=80=94=20and=20the=20rail=20keeps=20a=20curate?= =?UTF-8?q?d=20entry=20instead=20of=20generating=20one=20across=20a=20dive?= =?UTF-8?q?rged=20lineage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 15 +++++++++++++++ scripts/release.sh | 14 +++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62d81cfb..fc577c1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. + +### [10.4.12](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.11...v10.4.12) (2026-09-03) + +- Mixed-kind fields index exactly, arrays to 256, a drained loop is not a shutdown, and finds project from the column store +- fix(index): a metadata field holds every value kind it was written with — one posting column per (field, kind); an equality filter reads the query value's own kind, a range routes by its bounds; nothing is refused and nothing is silently dropped; an index written by the old shape opens unchanged (a128f0ed) +- fix(metadata): metadata arrays index up to 256 elements; a longer array refuses at write time by name (MetadataArrayTooLargeError) — a vector parked in metadata now throws; move it to `vector` (e435da78) +- fix(shutdown): beforeExit runs a non-closing flush only — a script that never calls close() exits with the writer lock on disk and no clean-shutdown marker, and the next open evicts the stale lock and folds the log, bounded; SIGTERM and SIGINT are unchanged (6baa4d7f) +- feat(find): field projection — find({fields}) and get({fields}) resolve scalars from the column store on every leg, including vector-leg finds; absent fields stay absent (ad0f493f) +- fix(find): orderBy is the order on every find path, not only the metadata-only one (5e720d17) +- fix(metadata): the legacy sparse range path orders values, or refuses by name — never ranks by hash (a7eb7f52) +- fix(close): a read-only brain writes nothing under `_system/` (f27a7776) +- fix(contract): the flush gate's internals are private, not doors (72c8ee6a) +- test(hygiene): the triple-intelligence correctness cases sit in the gate; the idle and connected-find pins name the brain they measure (28083981) +- ci(release): the rail writes its own wall entry into the shared releases repo — never hand-written again (adcb883e) + ### [10.4.11](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.9...v10.4.11) (2026-09-02) - ci: superseded pushes cancel their own runs (concurrency per ref) (6053f6d4) diff --git a/scripts/release.sh b/scripts/release.sh index 142fa06f..a9a1f6e9 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -159,9 +159,21 @@ CHANGELOG_ENTRY="### [${NEW_VERSION}](https://source.soulcraft.com/soulcraftlabs ${COMMITS} " +# A CURATED entry wins over the generated one. When a release is cut from a +# lineage that diverged from the previous tag (a candidate branch carrying +# main's history), `git log ..HEAD` lists every commit the tag never +# saw — old notes, already-shipped fixes under new hashes, merge commits — and a +# wall entry derived from it would misreport the release. If CHANGELOG.md +# already carries a `### [NEW_VERSION]` heading, it was written on purpose: +# keep it, and skip the generated prepend entirely. +CURATED_ENTRY=false +if grep -qE "^### \[${NEW_VERSION}\]" CHANGELOG.md 2>/dev/null; then + CURATED_ENTRY=true + echo -e "${YELLOW}CHANGELOG already carries a curated ### [${NEW_VERSION}] entry — keeping it, not generating one from commits${NC}" +fi # Prepend to CHANGELOG.md after header -if [ -f "CHANGELOG.md" ]; then +if [ "$CURATED_ENTRY" = false ] && [ -f "CHANGELOG.md" ]; then # Read header (first 4 lines) HEADER=$(head -n 4 CHANGELOG.md) # Read rest of file From 7e1ddee4f767950907942f162dbfacb55d3177e6 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:15:21 -0700 Subject: [PATCH 147/150] chore(release): 10.4.12 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3e3bf96d..c4757030 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.11", + "version": "10.4.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.11", + "version": "10.4.12", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 8676f8b7..649f2aaf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.11", + "version": "10.4.12", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", From 656d9f6f92e1ccdc29c9d86ffe1a172d5fc1219a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:18:57 -0700 Subject: [PATCH 148/150] test(hygiene): close every brain the remaining suites create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit id-normalization.test.ts's makeBrain() and degraded-reads-surfaced.test.ts's per-test brains had nothing tracking them — both now use a describe-scoped opened[] array drained by afterEach. find-hybrid-filter-before-hydrate.test.ts had two beforeAll-built brains (one per describe block) with no matching afterAll. multi-process-safety.test.ts and plugin-autodetect.test.ts/plugin.test.ts left a brain whose init() was expected to reject (a rejected init() still registers the instance in Brainy's global instance registry — the constructor does that unconditionally — so it still needs close() to deregister, or the process-level shutdown hooks never see the registry go idle for the rest of the run). --- .../find-hybrid-filter-before-hydrate.test.ts | 10 +++++++++- tests/integration/id-normalization.test.ts | 18 +++++++++++++++++- tests/integration/multi-process-safety.test.ts | 7 ++++++- .../brainy/degraded-reads-surfaced.test.ts | 10 +++++++++- tests/unit/plugin-autodetect.test.ts | 4 ++++ tests/unit/plugin.test.ts | 6 +++++- 6 files changed, 50 insertions(+), 5 deletions(-) diff --git a/tests/integration/find-hybrid-filter-before-hydrate.test.ts b/tests/integration/find-hybrid-filter-before-hydrate.test.ts index 3e74f5d8..7f326729 100644 --- a/tests/integration/find-hybrid-filter-before-hydrate.test.ts +++ b/tests/integration/find-hybrid-filter-before-hydrate.test.ts @@ -31,7 +31,7 @@ * never the legs. And the text leg is asked about the universe's ids only — * what it marshals is bounded by the universe, not by the store. */ -import { describe, it, expect, beforeAll, vi } from 'vitest' +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' import { Brainy } from '../../src/brainy' import { NounType, VerbType } from '../../src/types/graphTypes' import { rankIndicesByScore, reorderByIndices } from '../../src/utils/resultRanking' @@ -287,6 +287,10 @@ describe('hybrid find: filter before hydrate — the answer is unchanged', () => expect(typeof (brain as any).metadataIndex.getIdSetForFilter).not.toBe('function') }) + afterAll(async () => { + await brain.close() + }) + it('the fixture does not truncate the text leg — the universe covers every text match', async () => { const index = (brain as any).metadataIndex const textMatches = await index.getIdsForTextQuery(QUERY) @@ -553,6 +557,10 @@ describe('hybrid find: the text leg ranks inside the filter, not around it', () } }) + afterAll(async () => { + await brain.close() + }) + it('the old order let the filter consume the whole text leg', async () => { const index = (brain as any).metadataIndex const universe: string[] = await (brain as any).filterIdsBelted({ lane: 'alpha' }) diff --git a/tests/integration/id-normalization.test.ts b/tests/integration/id-normalization.test.ts index 1ea1a221..1eb14ab1 100644 --- a/tests/integration/id-normalization.test.ts +++ b/tests/integration/id-normalization.test.ts @@ -18,7 +18,7 @@ * All entities carry explicit 384-dim vectors so no test invokes the embedder. */ -import { describe, it, expect } from 'vitest' +import { describe, it, expect, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' import { v5, v7, isUUID } from '../../src/universal/uuid.js' @@ -37,8 +37,15 @@ async function makeBrain(): Promise { } describe('id normalization — transparent string-key round-trips', () => { + const opened: Brainy[] = [] + + afterEach(async () => { + for (const b of opened.splice(0)) await b.close().catch(() => {}) + }) + it('1. add() returns v5(key); get(key) and get(returnedId) both resolve; _originalId preserved', async () => { const brain = await makeBrain() + opened.push(brain) const returnedId = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) @@ -60,6 +67,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('2. relate() by string keys; related(key) and related({from:key}) return the edge to v5(toKey)', async () => { const brain = await makeBrain() + opened.push(brain) await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document }) @@ -85,6 +93,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('3. update() by string key reflects on get(key)', async () => { const brain = await makeBrain() + opened.push(brain) await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { role: 'admin' } }) await brain.update({ id: 'user-1', metadata: { role: 'owner' } }) @@ -98,6 +107,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('4. remove() by string key deletes; get(key) is null', async () => { const brain = await makeBrain() + opened.push(brain) await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) expect(await brain.get('user-1')).not.toBeNull() @@ -110,6 +120,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('5. find({ connected: { from: key } }) resolves the anchor key', async () => { const brain = await makeBrain() + opened.push(brain) await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) await brain.add({ id: 'doc-1', vector: vec(2), type: NounType.Document }) @@ -122,6 +133,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('6. transact() add+relate by string keys round-trips with consistent canonical ids', async () => { const brain = await makeBrain() + opened.push(brain) // Seed user-1 so the relate op has a target to point at. await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person }) @@ -149,6 +161,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('7. addMany() + relateMany() with string ids round-trip', async () => { const brain = await makeBrain() + opened.push(brain) const added = await brain.addMany({ items: [ @@ -175,6 +188,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('8. determinism: same key maps to same UUID — two adds upsert ONE entity, not two', async () => { const brain = await makeBrain() + opened.push(brain) const id1 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 1 } }) const id2 = await brain.add({ id: 'user-1', vector: vec(1), type: NounType.Person, metadata: { n: 2 } }) @@ -193,6 +207,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('9. valid-UUID passthrough: a real UUID is kept verbatim with NO _originalId', async () => { const brain = await makeBrain() + opened.push(brain) const realUuid = v7() const returnedId = await brain.add({ id: realUuid, vector: vec(5), type: NounType.Thing }) @@ -207,6 +222,7 @@ describe('id normalization — transparent string-key round-trips', () => { it('10. no-id add() mints a v7; newId() mints a v7', async () => { const brain = await makeBrain() + opened.push(brain) const autoId = await brain.add({ vector: vec(6), type: NounType.Thing }) expect(isUUID(autoId)).toBe(true) diff --git a/tests/integration/multi-process-safety.test.ts b/tests/integration/multi-process-safety.test.ts index 592d7969..dd1b8901 100644 --- a/tests/integration/multi-process-safety.test.ts +++ b/tests/integration/multi-process-safety.test.ts @@ -107,7 +107,11 @@ describe('Multi-process safety + read-only mode', () => { const blocked = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) await expect(blocked.init()).rejects.toThrow(/another writer holds/i) - // Don't track `blocked` for afterEach cleanup since init failed. + // A rejected init() still registered `blocked` in Brainy's global + // instance registry (the constructor does that unconditionally) — close() + // is safe to call even though init() never completed, and is what + // deregisters it (and, once idle, the process-level shutdown hooks). + await blocked.close().catch(() => {}) }) it('takes over a STALE foreign lock (dead PID + old heartbeat) and claims atomically', async () => { @@ -151,6 +155,7 @@ describe('Multi-process safety + read-only mode', () => { const err: any = await blocked.init().catch((e) => e) expect(err.code).toBe('BRAINY_WRITER_LOCKED') expect(err.lockInfo?.pid).toBe(otherPid) + await blocked.close().catch(() => {}) }) it('release drains an in-flight heartbeat — no phantom lock re-created after unlink', async () => { diff --git a/tests/unit/brainy/degraded-reads-surfaced.test.ts b/tests/unit/brainy/degraded-reads-surfaced.test.ts index 29a8a77c..004adeaa 100644 --- a/tests/unit/brainy/degraded-reads-surfaced.test.ts +++ b/tests/unit/brainy/degraded-reads-surfaced.test.ts @@ -19,13 +19,19 @@ import { prodLog } from '../../../src/utils/logger.js' const UUID = (suffix: string): string => `00000000-0000-4000-8000-0000000000${suffix}` describe('Finding 10 — degraded derived-index state is surfaced on reads', () => { + const opened: Brainy[] = [] + beforeEach(() => { process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' }) - afterEach(() => vi.restoreAllMocks()) + afterEach(async () => { + vi.restoreAllMocks() + for (const b of opened.splice(0)) await b.close().catch(() => {}) + }) it('checkHealth() reports adopt-forward degraded ids as unhealthy', async () => { const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + opened.push(brain) await brain.init() ;(brain as any)._indexDegradedIds.add(UUID('de')) @@ -37,6 +43,7 @@ describe('Finding 10 — degraded derived-index state is surfaced on reads', () it('find()/get() warn loudly while degraded, ONCE, then repairIndex() clears it', async () => { const warn = vi.spyOn(prodLog, 'warn').mockImplementation(() => {}) const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + opened.push(brain) await brain.init() await brain.add({ id: UUID('a1'), data: 'x', type: NounType.Document }) ;(brain as any)._indexRebuildFailed = new Error('rebuild boom') @@ -59,6 +66,7 @@ describe('Finding 10 — degraded derived-index state is surfaced on reads', () it('persistSingleOp records receipt.degraded (widened return type, not dropped)', async () => { const brain = new Brainy({ storage: { type: 'memory' }, dimensions: 384, requireSubtype: false }) + opened.push(brain) await brain.init() // Simulate a degraded receipt by wrapping the generation store's commitSingleOp. const gs: any = (brain as any).generationStore diff --git a/tests/unit/plugin-autodetect.test.ts b/tests/unit/plugin-autodetect.test.ts index 37c181ba..ee830c17 100644 --- a/tests/unit/plugin-autodetect.test.ts +++ b/tests/unit/plugin-autodetect.test.ts @@ -89,12 +89,14 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => { }) const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) await expect(brain.init()).rejects.toThrow(/installed but failed to load/) + await brain.close().catch(() => {}) }) it('installed but not a valid plugin (missing activate) → init() throws', async () => { stubImport(async () => ({ default: { name: '@soulcraft/cor' } })) // no activate() const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) await expect(brain.init()).rejects.toThrow(/not a valid Brainy plugin/) + await brain.close().catch(() => {}) }) it('installed but activation fails → init() throws (activateAll posture applies)', async () => { @@ -108,6 +110,7 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => { })) const brain: any = new Brainy({ requireSubtype: false, storage: { type: 'memory' }, silent: true }) await expect(brain.init()).rejects.toThrow(/failed to activate/) + await brain.close().catch(() => {}) }) it('plugins: [] and plugins: false → no probe at all (explicit opt-out)', async () => { @@ -132,5 +135,6 @@ describe('Guarded plugin auto-detection (plugins: undefined)', () => { silent: true }) await expect(brain.init()).rejects.toThrow(/listed in config\.plugins but could not be loaded/) + await brain.close().catch(() => {}) }) }) diff --git a/tests/unit/plugin.test.ts b/tests/unit/plugin.test.ts index f4064188..82543120 100644 --- a/tests/unit/plugin.test.ts +++ b/tests/unit/plugin.test.ts @@ -298,9 +298,10 @@ describe('Brainy plugin integration', () => { // must surface as a failed init(), NOT a silent degrade to the default // engine (the version-coupling guard; see plugin-version-coupling.test.ts). await expect(brain.init()).rejects.toThrow(/failed to activate|native module not found/) + await brain.close().catch(() => {}) }) - it('should use() return this for chaining', () => { + it('should use() return this for chaining', async () => { const plugin: BrainyPlugin = { name: 'chain-test', activate: async () => true @@ -309,5 +310,8 @@ describe('Brainy plugin integration', () => { const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) const result = brain.use(plugin) expect(result).toBe(brain) + // Never init()'d — the constructor still registered it in Brainy's global + // instance registry, so it still needs a close() to deregister. + await brain.close().catch(() => {}) }) }) From ba10aaf52ed14073509573950927c8f8714236e3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:32:17 -0700 Subject: [PATCH 149/150] test(hygiene): close two more brains found by a broadened rescan A second, structural pass of the honest scan (any local helper that constructs a Brainy directly, not just ones named like openBrain/makeBrain, plus support for new Brainy(...) generics) surfaced two more real leaks outside the first 93-file list: writer-lock-fencing.test.ts's `second` (a rejected-init() brain never pushed into the file's own tracked array) and plugin-version-coupling.test.ts's last case (a rejected-init() brain with no close at all). --- tests/integration/writer-lock-fencing.test.ts | 1 + tests/unit/plugin-version-coupling.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/integration/writer-lock-fencing.test.ts b/tests/integration/writer-lock-fencing.test.ts index e9f98dac..d5b82c30 100644 --- a/tests/integration/writer-lock-fencing.test.ts +++ b/tests/integration/writer-lock-fencing.test.ts @@ -61,6 +61,7 @@ describe('writer-lock fencing', () => { // Old rule: heartbeat-age eviction → silent takeover → split brain. // New rule: live PID = live writer; the second opener throws typed. const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) + brains.push(second) await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' }) }, 120000) diff --git a/tests/unit/plugin-version-coupling.test.ts b/tests/unit/plugin-version-coupling.test.ts index ffcc2a88..d4685ae2 100644 --- a/tests/unit/plugin-version-coupling.test.ts +++ b/tests/unit/plugin-version-coupling.test.ts @@ -143,5 +143,6 @@ describe('version coupling at init() — no silent fallback', () => { plugins: ['@soulcraft/this-package-does-not-exist-xyz'] }) await expect(brain.init()).rejects.toThrow(/could not be loaded|config\.plugins/) + await brain.close().catch(() => {}) }) }) From 6eb5e4483de50fc2099c4a6b61bf74c71a453e37 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 3 Sep 2026 09:38:26 -0700 Subject: [PATCH 150/150] test(hygiene): close the brain typeAware.bench.test.ts creates Excluded from the correctness gate (tests/performance/**, run only via npm run test:perf) but still leaked: brainMemory was created in a beforeEach with no matching afterEach. --- tests/performance/typeAware.bench.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/performance/typeAware.bench.test.ts b/tests/performance/typeAware.bench.test.ts index 72d96fe5..b1153662 100644 --- a/tests/performance/typeAware.bench.test.ts +++ b/tests/performance/typeAware.bench.test.ts @@ -17,7 +17,7 @@ * - Note limitations and edge cases */ -import { describe, it, expect, beforeEach } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Brainy } from '../../src/brainy.js' import { TypeAwareStorageAdapter } from '../../src/storage/adapters/typeAwareStorageAdapter.js' import { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js' @@ -67,6 +67,10 @@ describe('TypeAware Performance Benchmarks', () => { } }) + afterEach(async () => { + await brainMemory.close() + }) + it('should measure type-based query performance', async () => { // MEASURED: Query for one type (200 entities) const start = performance.now()