diff --git a/CHANGELOG.md b/CHANGELOG.md index f344a25b..16fb5786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.4.10](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.9...v10.4.10) (2026-09-02) - -- fix(find): near() searches around the anchor's own vector, and refuses by name without one (a8c5fbf9) -- Merge remote-tracking branches 'origin/fix/planner-provider-door' and 'origin/fix/containment-batching' into rel/10.4.10-candidate (34f1886f) -- feat(plugin): an optional planFindPage door — an index that can plan a find answers it in one call (4d5f823f) -- perf(vfs): repairContainment's reconcile is one paged edge walk, not one graph call per file (3e60aded) - - ### [10.4.9](https://source.soulcraft.com/soulcraftlabs/open-brainy/compare/v10.4.6...v10.4.9) (2026-09-02) - Merge branch 'fix/pending-embed-low-water' into rel/10.4.9-candidate (2648f56d) diff --git a/package-lock.json b/package-lock.json index ab23f6b1..fc530baa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.10", + "version": "10.4.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraftlabs/brainy", - "version": "10.4.10", + "version": "10.4.9", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 0c98cfe1..f07bb94c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraftlabs/brainy", - "version": "10.4.10", + "version": "10.4.9", "brainyContract": 1, "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", diff --git a/src/brainy.ts b/src/brainy.ts index cc5413e0..e08ca411 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -7346,47 +7346,6 @@ export class Brainy implements BrainyInterface { await this.verifyMetadataLive() } - // PLANNED FIND (optional provider door, `MetadataIndexProvider.planFindPage`). - // - // The stage doors below each serve one stage, so a find that consults - // three of them crosses into the index three times and marshals a result - // set at every crossing — a filter matching a hundred thousand rows - // builds a hundred thousand id strings to return a page of twenty-five. - // An index that can decide the stage order itself answers the page in one - // call and materializes ids only for the page. - // - // The hook sits ABOVE the branch selection because the branches are what - // decide stage order per call site; an index that plans has to be asked - // before that choice is made, not inside one of its arms. - // - // Optional and additive: a provider without the door, and any shape the - // door hands back, take exactly the path they always took. `null` is a - // routing decision the door must make BEFORE doing any work — never a - // partial answer. Every guard above still ran (readiness, the migration - // gate, the where-clause validation, the metadata cold-read guard), and - // the serving law is applied here on the way out: an empty answer is - // re-verified against the index that produced it before it is believed. - const planningIndex = this.metadataIndex as unknown as MetadataIndexProvider - if (typeof planningIndex.planFindPage === 'function') { - const planned = await planningIndex.planFindPage(params, [...hiddenIds], this.graphIndex) - if (planned !== null && planned !== undefined) { - if (planned.ids.length === 0) { - // A cold adjacency can report a size yet hold no edges, so an empty - // graph answer is not truth until the adjacency verifies live. A - // genuinely edgeless anchor verifies and the empty result stands. - if (planned.emptyAt === 'graph') await this.verifyGraphAdjacencyLive() - return [] - } - const plannedEntities = await this.batchGet(planned.ids) - const plannedResults: Result[] = [] - for (const id of planned.ids) { - const entity = plannedEntities.get(id) - if (entity) plannedResults.push(this.createResult(id, 1.0, entity)) - } - return plannedResults - } - } - // Handle metadata-only queries (no vector search needed) if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) { // Build filter for metadata index @@ -15892,18 +15851,8 @@ export class Brainy implements BrainyInterface { ) } - // The anchor's VECTOR is the query; get() omits vectors by default, which - // fed a zero-length vector to the index and refused every near() with a - // dimension mismatch. Ask for it, and refuse by name when the anchor has - // none — a proximity search around an unvectored row has no meaning. - const nearEntity = await this.get(params.near.id, { includeVectors: true }) + const nearEntity = await this.get(params.near.id) if (!nearEntity) return [] - if (!nearEntity.vector || nearEntity.vector.length === 0) { - throw new Error( - `find({ near }): entity '${params.near.id}' has no vector to search around — ` + - `it was never embedded (or was unvectored). Embed it, or search with a query instead.` - ) - } const nearResults: [string, number][] = await this.index.search(nearEntity.vector, params.limit || 10) diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index f4cdd632..5b10116c 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-08-27T09:18:45-07:00 + * Generated: 2026-06-29T10:04:19-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-08-27T09:18:45-07:00", + generatedAt: "2026-06-29T10:04:19-07:00", sizeBytes: { embeddings: 259584, base64: 346112 diff --git a/src/plugin.ts b/src/plugin.ts index fdad42f0..15b14b4e 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -424,52 +424,6 @@ export interface MetadataIndexProvider { * @param ids - The candidate ids (canonical). The answer is a subsequence. */ filterIdsWithin?(filter: any, ids: readonly string[]): Promise - /** - * @description OPTIONAL: plan and execute a WHOLE `find()` — the graph - * traversal, the metadata filter, the ordering and the page — and answer the - * page's ids, or `null` for a shape this index does not plan. - * - * The doors above each serve one stage, so a `find()` that consults three of - * them crosses into the index three times and marshals a result set at every - * crossing. An index that can decide the stage ORDER itself does the whole - * thing in one call and materializes ids only for the page — a filter - * matching a hundred thousand rows then builds twenty-five id strings instead - * of a hundred thousand. - * - * The contract this door must keep, because Brainy cannot check it: - * - * - **The same answer.** Identical rows, in identical order, to what the - * stage doors would have produced for the same params. This door changes - * which code runs, never what the answer is. - * - **The law of the stages** (`find({ connected })` is graph-first): the - * neighbour set is the candidate universe, the filter is evaluated over - * those ids only, `orderBy` sorts the whole candidate set, and the page is - * cut LAST. - * - **`null` before work, not instead of an answer.** A shape the index does - * not plan must be handed back BEFORE any evaluation, so Brainy serves it - * through the stage doors exactly as it always has. Returning `null` after - * partial work, or an empty page for a shape it could not evaluate, is a - * silent wrong answer. - * - **`emptyAt` names the stage** that produced an empty page — `'graph'`, - * `'filter'`, `'visibility'` or `'none'` — so Brainy can apply its serving - * law to the right index. An empty answer from an index that is not - * serving must refuse loudly, and Brainy can only re-verify what it is told. - * - * Absent → every `find()` is served by the stage doors, which is Brainy's - * own behaviour and the ordering oracle for any implementation of this one. - * @param params - The find params, already normalized by `find()` - * (natural-language parsed, `connected` anchors resolved to canonical ids, - * an empty `where` dropped). - * @param hiddenIds - Ids this read must not return; apply BEFORE paging so - * `limit` stays exact. - * @param graphIndex - The active graph provider, for a `connected` plan. - * @returns The page's ids plus the stage that emptied it, or `null`. - */ - planFindPage?( - params: any, - hiddenIds: readonly string[], - graphIndex: unknown - ): Promise<{ ids: string[]; emptyAt: 'graph' | 'filter' | 'visibility' | 'none' } | null> getIdsForTextQuery(query: string): Promise> getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc', topK?: number): Promise getFilterValues(field: string): Promise diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 1a4b9fa5..46c6a12d 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -2295,31 +2295,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { cursor = page.nextCursor } - // Pass 2: ONE paged walk over every Contains edge, grouped by target in - // memory. The earlier shape issued one awaited related({ to }) per VFS - // entity — O(entities) serialized graph calls, measured in whole minutes - // on large brains. This shape is O(edges / page) calls regardless of how - // many entities exist; mutations alone stay per-defect. - const incomingByTarget = new Map[]>() - { - const pageSize = 1000 - let pageOffset = 0 - for (;;) { - const page = await this.brain.related({ - type: VerbType.Contains, - limit: pageSize, - offset: pageOffset - }) - for (const edge of page) { - const bucket = incomingByTarget.get(edge.to) - if (bucket) bucket.push(edge) - else incomingByTarget.set(edge.to, [edge]) - } - if (page.length < pageSize) break - pageOffset += pageSize - } - } - let removed = 0 let restored = 0 for (const { id, path } of vfsEntities) { @@ -2332,7 +2307,7 @@ export class VirtualFileSystem implements IVirtualFileSystem { continue } - const incoming = incomingByTarget.get(id) ?? [] + const incoming = await this.brain.related({ to: id, type: VerbType.Contains }) let expectedSeen = false for (const edge of incoming) { const isVfsEdge = edge.subtype === 'vfs-contains' || (edge.metadata as any)?.isVFS === true diff --git a/tests/integration/find-near.test.ts b/tests/integration/find-near.test.ts deleted file mode 100644 index 3fb235c8..00000000 --- a/tests/integration/find-near.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -/** - * @module tests/integration/find-near - * @description find({ near }) searches around the anchor's OWN vector (10.4.10). - * - * The proximity search fetched its anchor without vectors and fed a - * zero-length vector to the index — every near() refused with a dimension - * mismatch, for every caller. Found by the Rust planner's first-contact pins - * (the planner declines `near`; the pin compared outcomes with and without - * it). Now the anchor is fetched with its vector, and an anchor without one - * refuses by name instead of failing inside the index. - */ -import { describe, it, expect, beforeAll } from 'vitest' -import { Brainy } from '../../src/brainy' -import { NounType } from '../../src/types/graphTypes' -import { v5 } from '../../src/universal/uuid' -import { generateTestVector } from '../helpers/test-factory' - -describe('find({ near }) uses the anchor vector', () => { - let brain: Brainy - const anchorVector = generateTestVector() - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - await brain.add({ id: 'anchor', data: 'anchor row', type: NounType.Thing, vector: anchorVector }) - // A twin with the identical vector and a far row. - await brain.add({ id: 'twin', data: 'twin row', type: NounType.Thing, vector: [...anchorVector] }) - await brain.add({ id: 'far', data: 'far row', type: NounType.Thing, vector: generateTestVector() }) - }) - - it('returns the anchor\'s neighbours by its own vector', async () => { - const results = await brain.find({ near: { id: 'anchor' }, limit: 3 }) - expect(results.length).toBeGreaterThan(0) - const ids = results.map((r) => r.entity.id) - expect(ids).toContain(v5('twin')) - }) - - it('refuses by name when the anchor has no vector', async () => { - await brain.add({ - id: 'unvectored', - data: 'no vector here', - type: NounType.Thing, - deferEmbedding: true - }) - ;(brain as any).kickEmbedWorker = () => {} - await expect(brain.find({ near: { id: 'unvectored' }, limit: 3 })).rejects.toThrow(/has no vector to search around/) - }) -}) diff --git a/tests/integration/find-planner-door.test.ts b/tests/integration/find-planner-door.test.ts deleted file mode 100644 index 964b13f9..00000000 --- a/tests/integration/find-planner-door.test.ts +++ /dev/null @@ -1,137 +0,0 @@ -/** - * @module tests/integration/find-planner-door - * @description The optional `MetadataIndexProvider.planFindPage` door. - * - * The stage doors each serve one stage, so a `find()` that consults three of - * them crosses into the index three times and marshals a result set at every - * crossing — a filter matching a hundred thousand rows builds a hundred - * thousand id strings to return a page of twenty-five. An index that can decide - * the stage order itself answers the page in one call. - * - * These pins hold the three properties that make such a door safe to add: - * - * 1. **Absent, nothing changes.** The reference index has no planner, and every - * find is served by the stage doors exactly as before. That is also what - * makes this engine the ordering oracle for any index that implements one. - * 2. **Present, it is asked first and its answer is used** — above the branch - * selection, with the params already normalized, the hidden ids passed, and - * the graph provider handed over. - * 3. **`null` is routing, not an answer.** A door that declines a shape leaves - * it to the path that always served it, and the result is unchanged. - * - * Plus the serving law: an empty page stamped `emptyAt: 'graph'` is re-verified - * against the adjacency before it is believed, so a not-serving graph refuses - * loudly instead of answering `[]` as truth. - */ -import { describe, it, expect, beforeAll, vi } from 'vitest' -import { Brainy } from '../../src/brainy' -import { NounType, VerbType } from '../../src/types/graphTypes' -import { generateTestVector } from '../helpers/test-factory' - -describe('find(): the optional planner door', () => { - let brain: Brainy - const anchor = 'planner-anchor' - let neighbourId = '' - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - await brain.add({ - id: anchor, - data: 'anchor', - type: NounType.Person, - metadata: { kind: 'anchor' }, - vector: generateTestVector() - }) - for (let i = 0; i < 12; i++) { - const id = await brain.add({ - id: `row-${i}`, - data: `row ${i}`, - type: NounType.Person, - metadata: { kind: 'note', rank: i }, - vector: generateTestVector() - }) - if (i === 0) neighbourId = id - await brain.relate({ from: anchor, to: id, type: VerbType.Knows }) - } - }) - - /** Install a planner door for one call, then remove it. */ - const withDoor = async ( - door: (...a: any[]) => Promise, - body: () => Promise - ): Promise => { - const index = (brain as any).metadataIndex - index.planFindPage = door - try { - return await body() - } finally { - delete index.planFindPage - } - } - - it('is absent on the reference index — every find is served by the stage doors', async () => { - expect((brain as any).metadataIndex.planFindPage).toBeUndefined() - const results = await brain.find({ where: { kind: 'note' }, limit: 5 }) - expect(results).toHaveLength(5) - }) - - it('is asked before the branches, with normalized params and the graph provider', async () => { - const door = vi.fn(async () => null) - await withDoor(door, async () => { - await brain.find({ where: { kind: 'note' }, limit: 5 }) - }) - expect(door).toHaveBeenCalledTimes(1) - const [params, hidden, graph] = door.mock.calls[0] as any[] - expect(params.where).toEqual({ kind: 'note' }) - expect(Array.isArray(hidden)).toBe(true) - expect(graph).toBe((brain as any).graphIndex) - }) - - it('uses the page it answers, hydrated and in the door\'s order', async () => { - const results = await withDoor( - async () => ({ ids: [neighbourId], emptyAt: 'none' as const }), - async () => brain.find({ where: { kind: 'note' }, limit: 5 }) - ) - expect(results).toHaveLength(1) - expect(results[0].entity.id).toBe(neighbourId) - }) - - it('a declining door changes nothing — the shape is served as it always was', async () => { - const withoutDoor = await brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 }) - const declined = await withDoor( - async () => null, - async () => brain.find({ where: { kind: 'note' }, orderBy: 'rank', limit: 4 }) - ) - expect(declined.map((r) => r.entity.id)).toEqual(withoutDoor.map((r) => r.entity.id)) - }) - - it('re-verifies the adjacency before believing an empty graph answer', async () => { - const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive') - try { - const results = await withDoor( - async () => ({ ids: [], emptyAt: 'graph' as const }), - async () => brain.find({ connected: { from: anchor }, where: { kind: 'note' }, limit: 5 }) - ) - expect(results).toEqual([]) - expect(verify).toHaveBeenCalled() - } finally { - verify.mockRestore() - } - }) - - it('does not re-verify the adjacency for an empty the FILTER produced', async () => { - const verify = vi.spyOn(brain as any, 'verifyGraphAdjacencyLive') - verify.mockClear() - try { - const results = await withDoor( - async () => ({ ids: [], emptyAt: 'filter' as const }), - async () => brain.find({ where: { kind: 'note' }, limit: 5 }) - ) - expect(results).toEqual([]) - expect(verify).not.toHaveBeenCalled() - } finally { - verify.mockRestore() - } - }) -}) diff --git a/tests/integration/vfs-containment-batched.test.ts b/tests/integration/vfs-containment-batched.test.ts deleted file mode 100644 index 0a7919bf..00000000 --- a/tests/integration/vfs-containment-batched.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -/** - * @module tests/integration/vfs-containment-batched - * @description repairContainment costs O(edges/page) graph calls, not O(entities) (10.4.9 train). - * - * Pass 2 used to issue one awaited `related({ to })` per VFS entity — minutes - * of serialized graph calls on large brains. Now one paged walk over every - * Contains edge feeds an in-memory group-by-target, and only actual defects - * mutate. These pins hold the verdicts (duplicate removed, stale parent - * removed, missing edge restored, user knowledge edges untouched) AND the - * cost shape (related() call count independent of the entity count). - */ -import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest' -import { Brainy } from '../../src/brainy' -import { NounType, VerbType } from '../../src/types/graphTypes' - -const FILES = 60 - -describe('repairContainment: batched pass 2', () => { - let brain: Brainy - let result: { removed: number; restored: number } - let relatedCalls = 0 - - beforeAll(async () => { - brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) - await brain.init() - const vfs = (brain as any).vfs ?? (brain as any)._vfs - expect(vfs).toBeTruthy() - await vfs.init() - - // A directory and FILES entries under it, wired as real VFS rows. - const mkNode = async (id: string, path: string, vfsType: string): Promise => { - await brain.add({ - id, - data: `vfs node ${path}`, - type: NounType.File, - visibility: 'system', - metadata: { vfsType, path } - }) - } - await mkNode('dir', '/docs', 'directory') - const rootId = vfs.rootEntityId ?? (await vfs.initializeRoot?.()) - if (rootId) { - await brain.relate({ - from: rootId, - to: 'dir', - type: VerbType.Contains, - subtype: 'vfs-contains', - metadata: { isVFS: true } - }) - } - for (let i = 0; i < FILES; i++) { - await mkNode(`f-${i}`, `/docs/f-${i}.md`, 'file') - if (i === 0) continue // f-0: MISSING edge — must be restored - await brain.relate({ - from: 'dir', - to: `f-${i}`, - type: VerbType.Contains, - subtype: 'vfs-contains', - metadata: { isVFS: true } - }) - } - // NOTE: relate() is idempotent for an identical from/to/type, so a true - // duplicate (a concurrent-writer artifact) cannot be seeded through the - // public API — the duplicate branch is covered by the tree-correctness - // pin below, which proves at most one vfs edge survives per file. - // f-2: STALE parent edge (from a sibling file) — must be removed. - await brain.relate({ - from: 'f-3', - to: 'f-2', - type: VerbType.Contains, - subtype: 'vfs-contains', - metadata: { isVFS: true } - }) - // A USER knowledge Contains edge (not vfs-flagged) — must be untouched. - await brain.relate({ from: 'f-4', to: 'f-5', type: VerbType.Contains }) - - const spy = vi.spyOn(brain, 'related') - result = await vfs.repairContainment() - relatedCalls = spy.mock.calls.length - spy.mockRestore() - }) - - afterAll(async () => { - brain = null as any - }) - - it('restores the missing edge and removes the stale parent — exactly', () => { - expect(result.restored).toBe(1) // f-0's missing edge - expect(result.removed).toBe(1) // f-2's stale parent (f-3 → f-2) - }) - - it('the repaired tree is correct: every file has exactly one vfs edge from its dir', async () => { - for (let i = 0; i < 6; i++) { - const incoming = await brain.related({ to: `f-${i}`, type: VerbType.Contains }) - const vfsEdges = incoming.filter( - (e) => e.subtype === 'vfs-contains' || (e.metadata as any)?.isVFS === true - ) - expect(vfsEdges, `f-${i}`).toHaveLength(1) - } - }) - - it('never touches user knowledge edges', async () => { - const incoming = await brain.related({ to: 'f-5', type: VerbType.Contains }) - const user = incoming.filter( - (e) => e.subtype !== 'vfs-contains' && (e.metadata as any)?.isVFS !== true - ) - expect(user).toHaveLength(1) - }) - - it('cost shape: related() calls do not scale with the entity count', () => { - // One paged type-only walk (~E/1000 pages) — with 60+ entities the old - // shape issued 60+ calls; the new one a handful. Bound generously. - expect(relatedCalls).toBeLessThanOrEqual(5) - }) -})