From eade6ff1bed1a1e40656061484929092718487e0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 10 Jun 2026 09:29:08 -0700 Subject: [PATCH 001/179] fix: mmap-vector backend capacity NaN at the provider FFI boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production deployment reported "[brainy] mmap-vector backend not wired (Capacity must be > 0); falling back to per-entity vector reads" on every cold start of populated brains under the native plugin — degrading vector recall to per-entity disk reads (8.3s cold starts at 685 entities, scaling linearly). Root cause: wireMmapVectorBackend computes its initial slot capacity as idMapper.size * 2. When the metadata index is provider-backed, getIdMapper() returns a façade exposing getInt/getOrAssign/getUuid but NO `size` property. `undefined * 2` is NaN, Math.max(NaN, 1024) stays NaN, and NaN coerces to 0 via ToUint32 at the provider's u32 FFI boundary — tripping the provider's "Capacity must be > 0" guard. The JS-only path never hit this because brainy's own EntityIdMapper has a real `size` getter. Fix, two independent layers: - wireMmapVectorBackend treats a missing/non-finite mapper size as 0, so the 1024-slot floor always holds (the file grows on demand past it). - MmapVectorBackend.open sanitizes the capacity (NaN/Infinity/non-positive → 16-slot floor) so no caller can ever hand the provider an invalid allocation size. Regression tests mimic the exact failure: a U32-coercing provider that rejects capacity 0 plus an idMapper façade without `size`. Pre-fix the NaN reached create() and threw; post-fix the backend wires with the floor capacity and round-trips vectors. 1464/1464 unit suite passing. --- src/brainy.ts | 10 +++- src/hnsw/mmapVectorBackend.ts | 8 +++- tests/unit/hnsw/mmap-vector-backend.test.ts | 52 +++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index d6bd7e39..e04df2c4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -9410,7 +9410,15 @@ export class Brainy implements BrainyInterface { if (!idMapper) return const dim = this.dimensions ?? 384 - const initialCapacity = Math.max(idMapper.size * 2, 1024) + // Provider-supplied id mappers (e.g. a native metadata index's façade) may + // not expose a `size` property. `undefined * 2` is NaN, and NaN coerces to + // 0 at the napi u32 boundary — the provider then rejects the create() with + // "Capacity must be > 0". Treat a missing/non-finite size as 0 so the + // 1024 floor always holds; the file grows on demand past it. + const mapperSize = Number.isFinite((idMapper as { size?: number }).size) + ? (idMapper as { size: number }).size + : 0 + const initialCapacity = Math.max(mapperSize * 2, 1024) try { const backend = await MmapVectorBackend.open( diff --git a/src/hnsw/mmapVectorBackend.ts b/src/hnsw/mmapVectorBackend.ts index fa9d4e37..cafcee4f 100644 --- a/src/hnsw/mmapVectorBackend.ts +++ b/src/hnsw/mmapVectorBackend.ts @@ -77,11 +77,17 @@ export class MmapVectorBackend { idMapper: EntityIdMapperProvider ): Promise { await mkdir(dirname(path), { recursive: true }) + // NaN/Infinity/non-positive capacities coerce to 0 at the provider's u32 + // FFI boundary ("Capacity must be > 0"). Sanitize here so no caller can + // ever hand the provider an invalid allocation size. + const capacity = Number.isFinite(initialCapacity) && initialCapacity >= 16 + ? Math.ceil(initialCapacity) + : 16 let store: VectorStoreMmapInstance try { store = provider.open(path) } catch { - store = provider.create(path, dim, Math.max(initialCapacity, 16)) + store = provider.create(path, dim, capacity) } return new MmapVectorBackend(store, idMapper) } diff --git a/tests/unit/hnsw/mmap-vector-backend.test.ts b/tests/unit/hnsw/mmap-vector-backend.test.ts index 257d50c4..5179ac9d 100644 --- a/tests/unit/hnsw/mmap-vector-backend.test.ts +++ b/tests/unit/hnsw/mmap-vector-backend.test.ts @@ -239,4 +239,56 @@ describe('MmapVectorBackend (2.4.0 #2 — wraps vectorStore:mmap provider)', () const backend2 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) expect(backend2.readByUuid('persisted')).toEqual([7, 7]) }) + + describe('capacity sanitization at the provider FFI boundary (7.31.3 regression)', () => { + /** + * Mimics cortex's NativeMmapVectorStore napi boundary: JS numbers are + * coerced ToUint32 (NaN → 0, Infinity → 0) before the Rust-side + * `capacity == 0` guard fires with "Capacity must be > 0". This is the + * exact production failure from a metadata-provider idMapper façade that + * exposes no `size` property: `undefined * 2` = NaN at the caller. + */ + class U32CoercingProvider extends MockMmapProvider { + lastCreateCapacity: number | null = null + override create(path: string, dim: number, capacity: number): VectorStoreMmapInstance { + const coerced = capacity >>> 0 // ECMA ToUint32 — what napi-rs does + this.lastCreateCapacity = coerced + if (coerced === 0) throw new Error('Capacity must be > 0') + return super.create(path, dim, coerced) + } + } + + it('NaN initialCapacity (idMapper façade without `size`) still wires with the floor capacity', async () => { + const strictProvider = new U32CoercingProvider() + const facadeWithoutSize = { + getInt: (uuid: string) => idMapper.getInt(uuid), + getOrAssign: (uuid: string) => idMapper.getOrAssign(uuid), + getUuid: (intId: number) => idMapper.getUuid(intId) + } as any + + // Pre-fix: NaN reached create(), coerced to 0, threw, and the backend + // never wired. Post-fix it must construct with the 16-slot floor. + const naiveCapacity = (facadeWithoutSize.size as number) * 2 // NaN + const backend = await MmapVectorBackend.open( + strictProvider, + path, + 4, + naiveCapacity, + facadeWithoutSize + ) + expect(strictProvider.lastCreateCapacity).toBeGreaterThanOrEqual(16) + + backend.writeByUuid('wired', [1, 2, 3, 4]) + expect(backend.readByUuid('wired')).toEqual([1, 2, 3, 4]) + }) + + it('Infinity and non-positive capacities are clamped to the floor', async () => { + for (const bad of [Infinity, -Infinity, 0, -5]) { + const strictProvider = new U32CoercingProvider() + const p = join(dir, `vectors-${String(bad)}.bin`) + await MmapVectorBackend.open(strictProvider, p, 2, bad, idMapper) + expect(strictProvider.lastCreateCapacity).toBe(16) + } + }) + }) }) From cfb051cc5a17fc508dcb9ce348e49e112c0c611a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 10 Jun 2026 09:31:48 -0700 Subject: [PATCH 002/179] chore(release): 7.31.3 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1df6e8ef..8370918e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.31.3](https://github.com/soulcraftlabs/brainy/compare/v7.31.2...v7.31.3) (2026-06-10) + +- fix: mmap-vector backend capacity NaN at the provider FFI boundary (eade6ff) + + ### [7.31.2](https://github.com/soulcraftlabs/brainy/compare/v7.31.1...v7.31.2) (2026-06-09) - docs: correct misleading SQ4 quantization comment in type definitions (89e4d81) diff --git a/package-lock.json b/package-lock.json index 04dc764b..71e7b4f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.31.2", + "version": "7.31.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.31.2", + "version": "7.31.3", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index cfd5eb5e..9e60997c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.31.2", + "version": "7.31.3", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 747ab974f31890ffca5822739be2c57161317aea Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 10 Jun 2026 10:50:46 -0700 Subject: [PATCH 003/179] fix: feature-detect setConnectionsCodec before wiring the connections codec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wireConnectionsCodec() called this.index.setConnectionsCodec(codec) unconditionally. Vector-index providers that don't keep per-node connection lists (single-file graph formats) have no such hook — the unconditional call forced them to carry a no-op shim just to survive brain.init(). Guard with a typeof check and skip silently: the delta-varint codec only applies to the JS HNSW connection layout, so providers without the hook lose nothing. Providers can now drop their shims. --- src/brainy.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/brainy.ts b/src/brainy.ts index e04df2c4..f7a102b7 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -9469,6 +9469,11 @@ export class Brainy implements BrainyInterface { const idMapper = this.metadataIndex.getIdMapper?.() if (!idMapper) return + // Feature-detect: vector-index providers without per-node connection + // lists (e.g. single-file graph formats) don't expose this hook. Skip + // silently — the codec only applies to the JS HNSW connection layout. + if (typeof this.index.setConnectionsCodec !== 'function') return + const codec = new ConnectionsCodec(provider, idMapper) this.index.setConnectionsCodec(codec) if (!this.config.silent) { From a8cbab6dd056be08546fda2b64a71fa51ba3c5f4 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 10 Jun 2026 10:51:03 -0700 Subject: [PATCH 004/179] chore(release): 7.31.4 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8370918e..7f3f2bc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.31.4](https://github.com/soulcraftlabs/brainy/compare/v7.31.3...v7.31.4) (2026-06-10) + +- fix: feature-detect setConnectionsCodec before wiring the connections codec (747ab97) + + ### [7.31.3](https://github.com/soulcraftlabs/brainy/compare/v7.31.2...v7.31.3) (2026-06-10) - fix: mmap-vector backend capacity NaN at the provider FFI boundary (eade6ff) diff --git a/package-lock.json b/package-lock.json index 71e7b4f0..4371a03d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.31.3", + "version": "7.31.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.31.3", + "version": "7.31.4", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index 9e60997c..631d2243 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.31.3", + "version": "7.31.4", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From a537b3664bd91937de3fb4c58113e9fd4c08f948 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 11 Jun 2026 09:17:15 -0700 Subject: [PATCH 005/179] fix: feature-detect setVectorBackend before wiring the mmap-vector backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production deployment on the native plugin reported the mmap-vector wiring failing one step past the 7.31.3 capacity fix: "this.index.setVectorBackend is not a function". Under the native plugin the active vector index is the provider's own class, which manages vector storage internally and exposes no external-backend hook — only Brainy's built-in JS HNSW index consumes one. Feature-detect the hook before doing any work (same pattern as 7.31.4's setConnectionsCodec guard), checked BEFORE opening the mmap file so no stray vector file is created for an index that will never read it. When the hook is absent the skip is logged at info level as expected behavior rather than surfacing as a scary error: the provider's index serves vectors its own way, and per-entity reads remain the designed fallback path. --- src/brainy.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/brainy.ts b/src/brainy.ts index f7a102b7..1c188c96 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -9400,6 +9400,24 @@ export class Brainy implements BrainyInterface { const provider = this.pluginRegistry.getProvider('vectorStore:mmap') if (!provider) return + // Feature-detect: only Brainy's own JS HNSW index consumes an external + // vector backend. A native vector-index provider manages its own vector + // storage and exposes no setVectorBackend hook — skip silently (same + // pattern as the setConnectionsCodec feature-detect). Checked BEFORE + // opening the backend so no mmap file is created that nothing will read. + const indexWithBackend = this.index as unknown as { + setVectorBackend?: (backend: MmapVectorBackend) => void + } + if (typeof indexWithBackend.setVectorBackend !== 'function') { + if (!this.config.silent) { + console.log( + '[brainy] mmap-vector backend not wired (vector index manages its own ' + + 'vector storage; no setVectorBackend hook) — per-entity reads in use' + ) + } + return + } + const storageWithBlob = this.storage as unknown as { getBinaryBlobPath?: (key: string) => string | null } From e5ec658fabe9b423a40f9ae259db95d9b28d7486 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 11 Jun 2026 09:17:31 -0700 Subject: [PATCH 006/179] chore(release): 7.31.5 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f3f2bc7..ea60972f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.31.5](https://github.com/soulcraftlabs/brainy/compare/v7.31.4...v7.31.5) (2026-06-11) + +- fix: feature-detect setVectorBackend before wiring the mmap-vector backend (a537b36) + + ### [7.31.4](https://github.com/soulcraftlabs/brainy/compare/v7.31.3...v7.31.4) (2026-06-10) - fix: feature-detect setConnectionsCodec before wiring the connections codec (747ab97) diff --git a/package-lock.json b/package-lock.json index 4371a03d..6e741861 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.31.4", + "version": "7.31.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.31.4", + "version": "7.31.5", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index 631d2243..039c521c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.31.4", + "version": "7.31.5", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 67e5fc8779cd82bd142f747f1152dbdfe867c849 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 11 Jun 2026 10:35:08 -0700 Subject: [PATCH 007/179] fix: remap reserved fields from update() metadata patches to their canonical location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit add({metadata: {confidence: 0.8}}) lifts reserved fields out of the metadata bag to their canonical top-level entity fields — teaching consumers that the metadata bag is a valid write path. update({metadata: {confidence: 0.33}}) then silently dropped the same shape: the patch value survived the metadata merge but was clobbered one expression later by the preserve-existing spread. No error, no warning, nothing written. A production consumer's confidence- evolution writes no-oped for weeks before being caught by reading values back. Fix: update() now normalizes the metadata patch before any enforcement or persistence logic runs, mirroring add()'s lift exactly: - confidence, weight, subtype — remapped to the top-level param unless the caller also passed that param explicitly (top-level wins). The remapped subtype flows through subtype-pairing enforcement like a top-level one. - noun, data, createdAt, updatedAt, service, createdBy, _rev — system-managed or owned by a dedicated param; dropped from the patch with a one-shot warning naming the correct write path (silent drop was the only wrong behavior here). Five regression tests pin the contract, including the production repro verbatim (add with metadata.confidence → top-level update → metadata-patch update → read-back) and the both-paths-supplied precedence case. 1475/1475 unit suite passing. --- src/brainy.ts | 71 ++++++++++- .../update-reserved-metadata-remap.test.ts | 117 ++++++++++++++++++ 2 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 tests/unit/brainy/update-reserved-metadata-remap.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 1c188c96..a8c6e8e4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -32,7 +32,7 @@ import { CommitBuilder } from './storage/cow/CommitObject.js' import { BlobStorage } from './storage/cow/BlobStorage.js' import { NULL_HASH } from './storage/cow/constants.js' import { createPipeline } from './streaming/pipeline.js' -import { configureLogger, LogLevel } from './utils/logger.js' +import { configureLogger, LogLevel, prodLog } from './utils/logger.js' import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' @@ -1527,6 +1527,64 @@ export class Brainy implements BrainyInterface { * }) * ``` */ + /** + * @description Normalize an update's metadata patch with respect to + * Brainy-reserved fields. `add()` lifts reserved fields out of `metadata` + * to their canonical top-level location; this applies the same contract to + * `update()` so the two write paths behave identically: + * - `confidence`, `weight`, `subtype` — remapped to the top-level param + * unless the caller also passed that param explicitly (top-level wins). + * - `noun`, `data`, `createdAt`, `updatedAt`, `service`, `createdBy`, + * `_rev` — system-managed or owned by a dedicated param; dropped from the + * patch with a one-shot warning that names the correct write path. + * @param params - The caller's update params (not mutated). + * @returns Params with reserved fields normalized out of `metadata`. + */ + private remapReservedMetadataFields(params: UpdateParams): UpdateParams { + if (!params.metadata || typeof params.metadata !== 'object') return params + + const { + noun, subtype, createdAt, updatedAt, confidence, weight, service, + data, createdBy, _rev, ...customMetadata + } = params.metadata as Record + + const hasReserved = + noun !== undefined || subtype !== undefined || createdAt !== undefined || + updatedAt !== undefined || confidence !== undefined || weight !== undefined || + service !== undefined || data !== undefined || createdBy !== undefined || + _rev !== undefined + if (!hasReserved) return params + + const warnDropped = (field: string, rightPath: string) => { + if (Brainy.warnedReservedFields.has(field)) return + Brainy.warnedReservedFields.add(field) + prodLog.warn( + `[brainy] update(): '${field}' is a reserved field and cannot be set ` + + `through a metadata patch — use ${rightPath}. The value was ignored. ` + + `(This warning is shown once per field per process.)` + ) + } + + if (noun !== undefined) warnDropped('noun', "the top-level 'type' param") + if (data !== undefined) warnDropped('data', "the top-level 'data' param") + if (createdAt !== undefined) warnDropped('createdAt', 'nothing — creation time is immutable') + if (updatedAt !== undefined) warnDropped('updatedAt', 'nothing — set automatically on every update') + if (service !== undefined) warnDropped('service', 'nothing — fixed at add() time') + if (createdBy !== undefined) warnDropped('createdBy', 'nothing — fixed at add() time') + if (_rev !== undefined) warnDropped('_rev', "the 'ifRev' param for optimistic concurrency") + + return { + ...params, + metadata: customMetadata as UpdateParams['metadata'], + ...(params.confidence === undefined && typeof confidence === 'number' && { confidence }), + ...(params.weight === undefined && typeof weight === 'number' && { weight }), + ...(params.subtype === undefined && typeof subtype === 'string' && { subtype }) + } + } + + /** One-shot registry for reserved-field warnings (per process). */ + private static warnedReservedFields = new Set() + async update(params: UpdateParams): Promise { this.assertWritable('update') await this.ensureInitialized() @@ -1534,6 +1592,17 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateUpdateParams(params) + // Reserved fields arriving via the metadata patch are remapped to their + // canonical top-level location, mirroring add()'s lift behavior. Before + // this fix the patch value survived the merge but was then clobbered by + // the preserve-existing spreads below — a silent no-op that consumers + // could only detect by reading values back. User-mutable reserved fields + // (confidence, weight, subtype) remap unless the same field was also + // passed top-level (top-level wins). System-managed fields (createdAt, + // updatedAt, _rev, noun, data, service, createdBy) cannot be set through + // a metadata patch and are dropped with a warning naming the right path. + params = this.remapReservedMetadataFields(params) + // Tracked-field vocabulary enforcement (Layer 2). Same as add() — the // metadata bag carries fields registered via trackField(), and subtype is // a tracked top-level candidate. diff --git a/tests/unit/brainy/update-reserved-metadata-remap.test.ts b/tests/unit/brainy/update-reserved-metadata-remap.test.ts new file mode 100644 index 00000000..122ed393 --- /dev/null +++ b/tests/unit/brainy/update-reserved-metadata-remap.test.ts @@ -0,0 +1,117 @@ +/** + * @module brainy/update-reserved-metadata-remap.test + * @description Regression tests for the reserved-field metadata-patch trap + * (7.31.6). `add({metadata: {confidence}})` lifts reserved fields to their + * canonical top-level location, but pre-7.31.6 `update({metadata: + * {confidence}})` silently dropped the same shape — the patch value survived + * the merge and was then clobbered by the preserve-existing spread. A + * production consumer's confidence-evolution writes no-oped for weeks before + * being caught by reading values back. + * + * Contract under test: update() remaps user-mutable reserved fields + * (confidence, weight, subtype) from the metadata patch to top-level + * (top-level param wins when both are present), and drops system-managed + * fields (createdAt, _rev, noun, data, ...) from patches with a warning. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' + +describe('update() reserved-field metadata-patch remap (7.31.6)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ storage: { type: 'memory' } }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('remaps metadata.confidence to the top-level field (the production repro)', async () => { + const id = await brain.add({ + type: 'concept', + subtype: 'general', + data: 'x', + metadata: { confidence: 0.8 } + }) + + // Top-level write works (always did) + await brain.update({ id, confidence: 0.42 }) + let entity = await brain.get(id) + expect(entity?.confidence).toBe(0.42) + + // Metadata-patch write — silently dropped pre-7.31.6, remapped now + await brain.update({ id, metadata: { confidence: 0.33 } }) + entity = await brain.get(id) + expect(entity?.confidence).toBe(0.33) + // The reserved key must not linger inside the metadata bag + expect((entity?.metadata as any)?.confidence).toBeUndefined() + }) + + it('remaps metadata.weight and metadata.subtype the same way', async () => { + const id = await brain.add({ + type: 'concept', + subtype: 'general', + data: 'y', + metadata: {} + }) + + await brain.update({ id, metadata: { weight: 0.7, subtype: 'specialized' } }) + const entity = await brain.get(id) + expect(entity?.weight).toBe(0.7) + expect(entity?.subtype).toBe('specialized') + expect((entity?.metadata as any)?.weight).toBeUndefined() + expect((entity?.metadata as any)?.subtype).toBeUndefined() + }) + + it('top-level param wins when both top-level and metadata-patch carry the field', async () => { + const id = await brain.add({ + type: 'concept', + subtype: 'general', + data: 'z', + metadata: { confidence: 0.5 } + }) + + await brain.update({ id, confidence: 0.9, metadata: { confidence: 0.1 } }) + const entity = await brain.get(id) + expect(entity?.confidence).toBe(0.9) + }) + + it('drops system-managed fields from patches without corrupting the entity', async () => { + const id = await brain.add({ + type: 'concept', + subtype: 'general', + data: 'w', + metadata: { keep: 'me' } + }) + const before = await brain.get(id) + + await brain.update({ + id, + metadata: { createdAt: 1, _rev: 999, noun: 'organization', other: 'applied' } + }) + const after = await brain.get(id) + + expect(after?.createdAt).toBe(before?.createdAt) // immutable + expect(after?.type).toBe('concept') // noun patch ignored + expect((after?.metadata as any)?.other).toBe('applied') // custom fields still merge + expect((after?.metadata as any)?.keep).toBe('me') + expect((after?.metadata as any)?._rev_).toBeUndefined() + }) + + it('custom (non-reserved) metadata patches are unaffected by the remap', async () => { + const id = await brain.add({ + type: 'concept', + subtype: 'general', + data: 'v', + metadata: { status: 'draft' } + }) + + await brain.update({ id, metadata: { status: 'reviewed', rating: 4.5 } }) + const entity = await brain.get(id) + expect((entity?.metadata as any)?.status).toBe('reviewed') + expect((entity?.metadata as any)?.rating).toBe(4.5) + }) +}) From 9b52629a0abb30702122c6ae2163ab26e7e52e6b Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 11 Jun 2026 10:35:26 -0700 Subject: [PATCH 008/179] chore(release): 7.31.6 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea60972f..bb4979fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.31.6](https://github.com/soulcraftlabs/brainy/compare/v7.31.5...v7.31.6) (2026-06-11) + +- fix: remap reserved fields from update() metadata patches to their canonical location (67e5fc8) + + ### [7.31.5](https://github.com/soulcraftlabs/brainy/compare/v7.31.4...v7.31.5) (2026-06-11) - fix: feature-detect setVectorBackend before wiring the mmap-vector backend (a537b36) diff --git a/package-lock.json b/package-lock.json index 6e741861..e33498bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.31.5", + "version": "7.31.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.31.5", + "version": "7.31.6", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index 039c521c..06093707 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.31.5", + "version": "7.31.6", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From ac29b0e650d9ec7cf674dae873e3d5732a736b45 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 11 Jun 2026 14:59:40 -0700 Subject: [PATCH 009/179] fix: vfs.rename() issues a metadata-only update + rollback of fresh adds removes them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes reported/found by downstream consumers: (1) vfs.rename() spread the entire fetched entity into brain.update(), forwarding a vector field that failed the 384-dimension validation when the entity was fetched without vectors — every rename threw. A rename is a path/metadata change, never a content change: the update is now metadata-only (no vector touched, no re-embedding). Regression test covers the verbatim consumer repro plus cross-directory moves and EEXIST. (2) AddToHNSWOperation.itemExists() probed an optional getItem capability with `await (index as any).getItem?.(id)` — `await undefined` resolves without throwing, so every item was reported as pre-existing and rollback of fresh adds never removed them, leaving phantom index entries after a failed transaction. The probe now feature-detects the method and defaults to false when absent; update flows pair this op with RemoveFromHNSWOperation whose own rollback restores the prior vector, so reverse-order rollback reconstructs the original state either way. 1479/1479 unit suite passing. --- src/transaction/operations/IndexOperations.ts | 21 ++++-- src/vfs/VirtualFileSystem.ts | 26 ++++--- .../unit/vfs/vfs-rename-metadata-only.test.ts | 68 +++++++++++++++++++ 3 files changed, 97 insertions(+), 18 deletions(-) create mode 100644 tests/unit/vfs/vfs-rename-metadata-only.test.ts diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 37db6227..891a9ac3 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -51,13 +51,26 @@ export class AddToHNSWOperation implements Operation { } /** - * Check if item exists in index + * Check if item exists in index. + * + * `getItem` is an optional provider capability that the built-in HNSW index + * does not implement. When the capability is missing the answer must be + * `false`, not `true`: `await undefined` resolves without throwing, so the + * old probe reported every item as pre-existing and rollback of fresh adds + * never removed them — leaving phantom index entries after a failed + * transaction. The safe default is to remove what this operation added; + * update flows pair this op with a RemoveFromHNSWOperation whose own + * rollback restores the prior vector, so reverse-order rollback + * reconstructs the original state either way. */ private async itemExists(id: string): Promise { + const index = this.index as HNSWIndex & { + getItem?: (id: string) => Promise + } + if (typeof index.getItem !== 'function') return false try { - // Try to get item - if exists, no error - await (this.index as any).getItem?.(id) - return true + const item = await index.getItem(id) + return item !== undefined && item !== null } catch { return false } diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 297f51e4..f47f8fbd 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -1664,17 +1664,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { if (err.code !== VFSErrorCode.ENOENT) throw err } - // Update entity metadata - const updatedEntity = { - ...entity, - metadata: { - ...entity.metadata, - path: newPath, - name: this.getBasename(newPath), - modified: Date.now() - } - } - // Update parent relationships if needed const oldParentPath = this.getParentPath(oldPath) const newParentPath = this.getParentPath(newPath) @@ -1700,10 +1689,19 @@ export class VirtualFileSystem implements IVirtualFileSystem { } } - // Update the entity + // A rename is a path/metadata change, never a content change — issue a + // metadata-only update. Spreading the whole entity here used to forward + // its vector field into update(), which failed dimension validation when + // the entity was fetched without vectors (and would needlessly touch the + // vector index when it wasn't). await this.brain.update({ - ...updatedEntity, - id: entityId + id: entityId, + metadata: { + ...entity.metadata, + path: newPath, + name: this.getBasename(newPath), + modified: Date.now() + } }) // Update path cache diff --git a/tests/unit/vfs/vfs-rename-metadata-only.test.ts b/tests/unit/vfs/vfs-rename-metadata-only.test.ts new file mode 100644 index 00000000..e9718010 --- /dev/null +++ b/tests/unit/vfs/vfs-rename-metadata-only.test.ts @@ -0,0 +1,68 @@ +/** + * @module vfs/vfs-rename-metadata-only.test + * @description Regression test for the vfs.rename() vector-dimension crash + * (7.31.7). rename() used to spread the entire fetched entity into + * brain.update(), forwarding a vector field that failed the 384-dimension + * validation when the entity was fetched without vectors. A rename is a + * path/metadata change — the update must be metadata-only. + * + * Production repro (verbatim from the consumer report): + * await brain.vfs.writeFile('/a.txt', 'x') + * await brain.vfs.rename('/a.txt', '/b.txt') // threw pre-7.31.7 + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/index.js' + +describe('vfs.rename() issues a metadata-only update (7.31.7)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ storage: { type: 'memory' } }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + it('renames a file without touching vectors (the production repro)', async () => { + await brain.vfs.writeFile('/a.txt', 'x') + await brain.vfs.rename('/a.txt', '/b.txt') + + expect(await brain.vfs.exists('/b.txt')).toBe(true) + expect(await brain.vfs.exists('/a.txt')).toBe(false) + const content = await brain.vfs.readFile('/b.txt') + expect(content.toString()).toBe('x') + }) + + it('preserves file content and custom metadata across rename', async () => { + await brain.vfs.writeFile('/notes/draft.md', 'hello world') + await brain.vfs.rename('/notes/draft.md', '/notes/final.md') + + const content = await brain.vfs.readFile('/notes/final.md') + expect(content.toString()).toBe('hello world') + // The renamed entity's metadata carries the new basename + const stat = await brain.vfs.stat('/notes/final.md') + expect(stat.size).toBeGreaterThan(0) + expect(await brain.vfs.exists('/notes/draft.md')).toBe(false) + }) + + it('moves a file across directories', async () => { + await brain.vfs.mkdir('/src') + await brain.vfs.mkdir('/dest') + await brain.vfs.writeFile('/src/file.txt', 'moving') + await brain.vfs.rename('/src/file.txt', '/dest/file.txt') + + expect(await brain.vfs.exists('/dest/file.txt')).toBe(true) + expect(await brain.vfs.exists('/src/file.txt')).toBe(false) + const content = await brain.vfs.readFile('/dest/file.txt') + expect(content.toString()).toBe('moving') + }) + + it('throws EEXIST when the target already exists', async () => { + await brain.vfs.writeFile('/one.txt', '1') + await brain.vfs.writeFile('/two.txt', '2') + await expect(brain.vfs.rename('/one.txt', '/two.txt')).rejects.toThrow(/exists/i) + }) +}) From 4f8159c572983da9009e93bd630ad12a75d552bf Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 11 Jun 2026 15:00:26 -0700 Subject: [PATCH 010/179] chore(release): 7.31.7 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4979fb..5e1b29f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.31.7](https://github.com/soulcraftlabs/brainy/compare/v7.31.6...v7.31.7) (2026-06-11) + +- fix: vfs.rename() issues a metadata-only update + rollback of fresh adds removes them (ac29b0e) + + ### [7.31.6](https://github.com/soulcraftlabs/brainy/compare/v7.31.5...v7.31.6) (2026-06-11) - fix: remap reserved fields from update() metadata patches to their canonical location (67e5fc8) diff --git a/package-lock.json b/package-lock.json index e33498bd..6cf3d433 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.31.6", + "version": "7.31.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.31.6", + "version": "7.31.7", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index 06093707..c9ab40a1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.31.6", + "version": "7.31.7", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 3f8e0971a2d671668f1b2983a3e71e3fd3744e59 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 16 Jun 2026 08:46:39 -0700 Subject: [PATCH 011/179] fix: query-cap memory misread (MemAvailable + floor) + rootDirectory getter for native mmap fast-path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two platform-wide production fixes for consumers on bare VMs / mmap-filesystem storage. BUG A — auto maxQueryLimit collapsed to ~1000 on healthy VMs → 500s on legitimate queries. getAvailableMemory() read os.freemem() (kernel MemFree, which excludes reclaimable page cache and reads as tens of MB on a page-cache-heavy mmap box). Now reads /proc/meminfo MemAvailable (the `free -h` figure), falling back to os.freemem() only off-Linux; auto-detected caps (container/free branches) are floored at 10k so a misread can't collapse them. Explicit maxQueryLimit/reservedQueryMemory are honored as-is. BUG B — native mmap vector fast-path never engaged (per-entity reads → 57s cold start on a 283MB brain). FileSystemStorage now exposes a public `rootDirectory` getter; the native vector provider feature-detects it to enable its memory-mapped graph path. brainy stored it as the protected `rootDir`, so the gate silently failed. Self-heals after the first post-upgrade flush writes the mmap file. Regression tests: auto-cap floor (container/free/explicit-override) + the rootDirectory getter. Build clean; full suite 1483 green. --- RELEASES.md | 23 +++++++++++ src/storage/adapters/fileSystemStorage.ts | 13 ++++++ src/utils/paramValidation.ts | 41 ++++++++++++++++--- .../fileSystemStorage-rootDirectory.test.ts | 28 +++++++++++++ tests/unit/utils/memoryLimits.test.ts | 28 +++++++++++++ 5 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 tests/unit/storage/fileSystemStorage-rootDirectory.test.ts diff --git a/RELEASES.md b/RELEASES.md index 58b20bd2..81ef906a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,29 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so --- +## v7.31.8 — 2026-06-16 + +**Affected products:** every consumer on a bare VM or `mmap-filesystem` storage — **upgrade recommended** (Memory, Workshop on 7.31.x; Venue on 7.28.x). Two platform-wide production fixes. Drop-in; no API or data changes. + +### Fix 1 — auto query-limit cap no longer collapses on a healthy VM (the 500s) + +`new Brainy()`'s auto-configured `maxQueryLimit` could collapse to ~1000 on a perfectly healthy box, throwing `exceeds the auto-configured query limit` on legitimate queries (e.g. `find({ limit: 2400 })`) → HTTP 500s. Root cause: detection read `os.freemem()` (kernel MemFree), which **excludes reclaimable page cache** — on a long-running box using mmap-filesystem storage the page cache grows until MemFree is a thin sliver (tens of MB) even though several GB are actually available, so `floor(freemem / 25KB)·1000` floored the cap to ~1000. + +- Memory detection now reads **`/proc/meminfo` `MemAvailable`** (the figure `free -h` shows — counts reclaimable cache), falling back to `os.freemem()` only off-Linux. +- A **10,000 floor** on auto-detected caps (container / free-memory branches) so a detection miss can never silently collapse the cap. Explicit `maxQueryLimit` / `reservedQueryMemory` settings are honored as-is (never floored). + +No action beyond upgrading. A `MEMORY_LIMIT` override (if you set one as a workaround) is still honored. + +### Fix 2 — native mmap vector fast-path re-engages (cold-start 57s → instant) + +`FileSystemStorage` now exposes a public `rootDirectory` getter. The optional native vector provider feature-detects `storage.rootDirectory` to enable its memory-mapped graph fast path; brainy stored the path as the protected `rootDir`, so the gate silently failed and every cold start rebuilt the index via per-entity disk reads (~57s for a 283 MB brain on a measured deployment). With the getter, the mmap path engages and cold start becomes a near-instant memory-mapped load. + +Self-heals after the first post-upgrade flush writes the mmap file — the first restart after upgrading rebuilds once, then every subsequent boot is fast. Benefits the full deployed matrix (the gate exists in the native provider versions Memory/Workshop and Venue run). + +**Upgrade:** bump `@soulcraft/brainy` to `7.31.8`. No code or data migration. + +--- + ## v7.31.2 — 2026-06-09 **Affected products:** none in behaviour; affects anyone reading Brainy's TypeScript diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index e86e1a6d..3b11cf65 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -81,6 +81,19 @@ export class FileSystemStorage extends BaseStorage { private readonly MAX_SHARDS = 256 // Hex range: 00-ff private cachedShardingDepth: number = this.SHARDING_DEPTH // Always use fixed depth protected rootDir: string + + /** + * Public read-only alias for the storage root path. Native vector plugins + * (e.g. `@soulcraft/cortex`) feature-detect `storage.rootDirectory` to enable + * their memory-mapped vector fast path; brainy stores it as the protected + * `rootDir`, so without this getter the native mmap backend stays un-wired and + * the index rebuilds via slow per-entity disk reads on cold start. Exposing + * the alias lets the native backend memory-map its index file instead. + */ + get rootDirectory(): string { + return this.rootDir + } + private nounsDir!: string private verbsDir!: string private metadataDir!: string diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index cb26baaf..86b06ea5 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -32,6 +32,24 @@ const getSystemMemory = (): number => { } const getAvailableMemory = (): number => { + // Prefer /proc/meminfo `MemAvailable` — it counts reclaimable page cache as + // available, which `os.freemem()` (kernel MemFree) does NOT. On a long-running + // box using mmap-filesystem storage, the page cache grows until MemFree is a + // thin sliver (tens of MB) even though several GB are actually available; the + // freemem reading then collapses the auto query-limit cap to ~1000 and 500s + // legitimate queries. MemAvailable reflects the real figure `free -h` shows. + if (fs) { + try { + const meminfo = fs.readFileSync('/proc/meminfo', 'utf8') as string + const match = meminfo.match(/^MemAvailable:\s+(\d+)\s*kB/m) + if (match) { + const bytes = parseInt(match[1], 10) * 1024 + if (bytes > 0) return bytes + } + } catch { + // Not Linux / no procfs — fall through to freemem. + } + } if (os) { return os.freemem() } @@ -137,6 +155,11 @@ const getContainerMemoryLimit = (): number | null => { * the throw tier still fires before OOM territory (72 K+). */ const MAX_LIMIT_KB_PER_RESULT = 25 +// Floor for AUTO-DETECTED caps (container/free-memory branches). A memory +// misread must never silently collapse the cap below a practical safety value +// and 500 legitimate queries; explicit operator settings (maxQueryLimit / +// reservedQueryMemory) are honored as-is and bypass this floor. +const MIN_AUTO_QUERY_LIMIT = 10000 /** * One-time-per-call-site warning dedup. Keyed on the caller location returned @@ -230,9 +253,12 @@ export class ValidationConfig { // Reserve 25% for query operations const queryMemory = this.detectedContainerLimit * 0.25 - this.maxLimit = Math.min( - 100000, - Math.floor(queryMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000 + this.maxLimit = Math.max( + MIN_AUTO_QUERY_LIMIT, + Math.min( + 100000, + Math.floor(queryMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000 + ) ) this.limitBasis = 'containerMemory' @@ -246,9 +272,12 @@ export class ValidationConfig { // Priority 4: Free memory (fallback, current behavior) const availableMemory = getAvailableMemory() - this.maxLimit = Math.min( - 100000, - Math.floor(availableMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000 + this.maxLimit = Math.max( + MIN_AUTO_QUERY_LIMIT, + Math.min( + 100000, + Math.floor(availableMemory / (1024 * 1024 * MAX_LIMIT_KB_PER_RESULT)) * 1000 + ) ) this.limitBasis = 'freeMemory' diff --git a/tests/unit/storage/fileSystemStorage-rootDirectory.test.ts b/tests/unit/storage/fileSystemStorage-rootDirectory.test.ts new file mode 100644 index 00000000..83b0c2f5 --- /dev/null +++ b/tests/unit/storage/fileSystemStorage-rootDirectory.test.ts @@ -0,0 +1,28 @@ +/** + * Regression: FileSystemStorage must expose a public `rootDirectory` getter. + * + * Native vector plugins (e.g. @soulcraft/cortex's NativeHNSWWrapper) feature-detect + * `storage.rootDirectory` to enable their memory-mapped vector fast path. Brainy + * stores the path as the protected `rootDir`; without this public alias the native + * mmap backend stayed un-wired and the index rebuilt via slow per-entity disk reads + * on cold start (BRAINY-MMAP-VECTOR-HOOK). The getter is the load-bearing line, so + * guard it explicitly. + */ +import { describe, it, expect } from 'vitest' +import { mkdtempSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' + +describe('FileSystemStorage.rootDirectory (native mmap-backend hook)', () => { + it('exposes the constructor root path via a public rootDirectory getter', () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-rootdir-')) + try { + const storage = new FileSystemStorage(dir) + // The exact property a native vector backend reads to engage its mmap path. + expect((storage as unknown as { rootDirectory: string }).rootDirectory).toBe(dir) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/unit/utils/memoryLimits.test.ts b/tests/unit/utils/memoryLimits.test.ts index aca6d91c..e473903d 100644 --- a/tests/unit/utils/memoryLimits.test.ts +++ b/tests/unit/utils/memoryLimits.test.ts @@ -399,4 +399,32 @@ describe('Memory Limits - Container Detection & Smart Calculation', () => { await brain.close() }) }) + + // Regression: a memory MISREAD (tiny detected limit, or os.freemem reading a + // sliver on a page-cache-heavy mmap box) must NOT collapse the auto cap to + // ~1000 and 500 legitimate queries. Auto-detected caps are floored at 10k; + // explicit operator settings still pass through unfloored. + describe('Auto-cap floor (BUG: querycap collapse on healthy VM)', () => { + it('floors the container-derived cap so a tiny/misread limit cannot collapse it', () => { + // 50 MB → queryMemory 12.5 MB → floor(12.5/25)·1000 = 0 pre-fix. + process.env.MEMORY_LIMIT = String(50 * 1024 * 1024) + const config = ValidationConfig.getInstance() + expect(config.limitBasis).toBe('containerMemory') + expect(config.maxLimit).toBe(10000) // floored, not 0/1000 + }) + + it('floors the free-memory-derived cap as well', () => { + // No container env → freeMemory branch. On any real box this is well + // above the floor; the guarantee is it can never drop below it. + const config = ValidationConfig.getInstance() + expect(config.limitBasis).toBe('freeMemory') + expect(config.maxLimit).toBeGreaterThanOrEqual(10000) + }) + + it('does NOT floor an explicit (small) operator override', () => { + const config = ValidationConfig.getInstance({ maxQueryLimit: 500 }) + expect(config.limitBasis).toBe('override') + expect(config.maxLimit).toBe(500) // explicit intent respected, not floored + }) + }) }) From 89c6d043ba689054eb2be1e8236ab4f785b8f492 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 16 Jun 2026 08:48:06 -0700 Subject: [PATCH 012/179] chore(release): 7.31.8 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e1b29f5..4eeee7c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.31.8](https://github.com/soulcraftlabs/brainy/compare/v7.31.7...v7.31.8) (2026-06-16) + +- fix: query-cap memory misread (MemAvailable + floor) + rootDirectory getter for native mmap fast-path (3f8e097) + + ### [7.31.7](https://github.com/soulcraftlabs/brainy/compare/v7.31.6...v7.31.7) (2026-06-11) - fix: vfs.rename() issues a metadata-only update + rollback of fresh adds removes them (ac29b0e) diff --git a/package-lock.json b/package-lock.json index 6cf3d433..c479349a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.31.7", + "version": "7.31.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.31.7", + "version": "7.31.8", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index c9ab40a1..31ad3502 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.31.7", + "version": "7.31.8", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From a408d3799dee8779c4f8b140496eaac092072dfa Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 16 Jun 2026 15:52:42 -0700 Subject: [PATCH 013/179] feat: portable graph export()/import() (BackupData v1) on brain.data() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit brain.data() now serializes part or all of a brain to a versioned, portable JSON document (BackupData) and restores it — the path for partial backups, cross-environment moves, and 7.x→8.0 migration. - export(selector?, options?): select by ids, collection (+transitive Contains), connected neighbourhood, vfsPath subtree, predicate, or whole brain; structural and predicate selectors compose. Options: includeVectors, includeContent (VFS blobs), includeSystem, edges ('induced'|'incident'|'none'). - import(backup, options?): onConflict 'merge' (dedup-by-id) | 'replace' | 'skip'; reembed 'auto' (re-embed from data when no vector carried) | 'never'; remapIds to clone a subgraph under fresh ids. - BackupData v1: format/formatVersion/brainyVersion/createdAt/embedding/entities/ relations/blobs?/danglingIds?/stats. Reserved fields (subtype, data, confidence, weight, service) top-level; metadata custom-only. Current-state, no generations. - Replaces the prior flat-entity export() (dropped relations) with a graph-complete document. Distinct from brain.import(file) ingestion, which is unchanged. - Export BackupData/BackupEntity/BackupRelation/ExportSelector/ExportOptions/ ImportOptions/ImportResult/DataAPI from the package root. CLI `brainy export` now writes a BackupData document. Guide: docs/guides/backup-and-export.md. Tests: tests/unit/api/data-backup.test.ts. --- RELEASES.md | 47 + docs/api/README.md | 29 +- docs/architecture/storage-architecture.md | 24 +- docs/augmentations/EXAMPLES.md | 8 +- docs/guides/export-and-import.md | 173 ++++ docs/guides/framework-integration.md | 6 +- src/api/DataAPI.ts | 1082 +++++++++++++++------ src/brainy.ts | 7 +- src/cli/commands/core.ts | 76 +- src/index.ts | 12 + tests/unit/api/data-backup.test.ts | 311 ++++++ 11 files changed, 1393 insertions(+), 382 deletions(-) create mode 100644 docs/guides/export-and-import.md create mode 100644 tests/unit/api/data-backup.test.ts diff --git a/RELEASES.md b/RELEASES.md index 81ef906a..ea16c8d2 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,53 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so --- +## v7.32.0 — 2026-06-16 + +**Affected products:** anyone who needs a **portable graph backup**, a partial export, or a +**7.x → 8.0 migration path**. Purely additive — no API removed, no data change. Drop-in. + +### New — portable graph `export()` / `import()` on `brain.data()` + +`brain.data()` gains a portable backup/restore pair that serializes part or all of a brain +to a single versioned JSON document (`BackupData`) and restores it. This is the format that +embeds inside portable artifacts (e.g. a serialized subgraph carried by a higher-level file +type) and the recommended way to move a graph between environments or **upgrade an existing +7.x brain into an 8.0 brain**. + +```typescript +const data = await brain.data() + +const backup = await data.export() // whole brain → BackupData +const subset = await data.export({ ids }, { includeVectors: true }) +await otherBrain.data().then(d => d.import(subset, { onConflict: 'merge' })) +``` + +- **Selectors** — `export(selector?, options?)` picks the node set: `{ ids }`, `{ collection }` + (alias `memberOf`) + transitive `Contains` members, `{ connected: { from, depth, verbs?, direction? } }`, + `{ vfsPath, recursive? }`, a predicate (`{ type, subtype, where, service }`), or the whole brain + (omit). Structural + predicate selectors **compose**. +- **Options** — `includeVectors` (default off → re-embed on import), `includeContent` (VFS file + bytes in `blobs`), `includeSystem` (include the VFS root etc.), `edges` (`'induced'` default / + `'incident'` / `'none'`). +- **Import** — `import(backup, options?)` → `{ imported, merged, skipped, reembedded, blobsWritten, errors }`. + `onConflict` defaults to `'merge'` (dedup-by-id, so you can assemble many documents into one + graph); `reembed` defaults to `'auto'` (re-embed from `data` when no vector is carried); + `remapIds` clones a subgraph under fresh ids. +- **Format** — `{ format:'brainy-backup', formatVersion:1, brainyVersion, createdAt, embedding, + entities, relations, blobs?, danglingIds?, stats }`. Entities carry `subtype` and the standard + reserved fields at the top level; `metadata` is custom-only. **Current-state** (no generation + history). `formatVersion` gates cross-version import (a 7.x document imports into 8.0). +- **Types** — `BackupData`, `BackupEntity`, `BackupRelation`, `ExportSelector`, `ExportOptions`, + `ImportOptions`, `ImportResult`, and the `DataAPI` class are now exported from the package root. + +This replaces the previous `brain.data().export()` flat-entity array (which dropped relations) +with a graph-complete document. Distinct from `brain.import(file)` (CSV/PDF/Excel/JSON ingestion), +which is unchanged. The CLI `brainy export` now writes a `BackupData` document (json/jsonl/csv). + +Guide: `docs/guides/export-and-import.md`. Tested in `tests/unit/api/data-backup.test.ts`. + +--- + ## v7.31.8 — 2026-06-16 **Affected products:** every consumer on a bare VM or `mmap-filesystem` storage — **upgrade recommended** (Memory, Workshop on 7.31.x; Venue on 7.28.x). Two platform-wide production fixes. Drop-in; no API or data changes. diff --git a/docs/api/README.md b/docs/api/README.md index 69df13fb..595a3bbd 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -1765,20 +1765,33 @@ await brain.import('https://api.example.com/data.json') --- -### Export & Snapshots +### Export & Import (portable backup) + +`brain.data()` exposes a portable graph backup/restore API. `export(selector?, options?)` +serializes part or all of the graph to a versioned, portable `BackupData` document; +`import(backup, options?)` restores it (dedup-by-id merge by default, re-embedding when +vectors are absent). ```typescript -// Export to file -await brain.export('/path/to/backup.brainy') +const data = await brain.data() -// Create instant snapshot using COW fork -await brain.fork('backup-2025-01-19') +// Whole brain → a portable BackupData document +const backup = await data.export() -// Time-travel to specific commit -const snapshot = await brain.asOf(commitId) -const entities = await snapshot.find({ limit: 100 }) +// Just one workbench's members, with vectors, then restore elsewhere (merge by id) +const subset = await data.export({ ids }, { includeVectors: true }) +await otherBrain.data().then(d => d.import(subset, { onConflict: 'merge' })) + +// A collection + its children; a connected neighbourhood; a VFS subtree (+ bytes) +await data.export({ collection: collectionId }) +await data.export({ connected: { from: id, depth: 2 } }) +await data.export({ vfsPath: '/docs' }, { includeContent: true }) ``` +The format is versioned (`formatVersion`) and current-state (no generation history) — see +the **[Export & Import guide →](../guides/export-and-import.md)**. This is distinct from +`brain.import(file)` (CSV/PDF/Excel/JSON ingestion). + --- ## Configuration diff --git a/docs/architecture/storage-architecture.md b/docs/architecture/storage-architecture.md index 5c71d50c..fb9c0025 100644 --- a/docs/architecture/storage-architecture.md +++ b/docs/architecture/storage-architecture.md @@ -457,23 +457,19 @@ await brain.storage.withLock('resource-id', async () => { ### Export Data ```typescript -// Export entire database -const backup = await brain.export({ - format: 'json', - includeVectors: true, - includeIndexes: false -}) +// Export the whole brain to a portable BackupData document +const backup = await brain.data().then(d => d.export(undefined, { includeVectors: true })) ``` ### Import Data ```typescript -// Import from backup -await brain.import(backup, { - mode: 'merge', // or 'replace' - validateSchema: true -}) +// Restore a BackupData document (dedup-by-id merge by default) +await brain.data().then(d => d.import(backup, { onConflict: 'merge' })) ``` +See the [Export & Import guide](../guides/export-and-import.md) for partial exports +(by id, collection, connected neighbourhood, VFS subtree, or predicate). + ### Storage Migration ```typescript // Migrate between storage types @@ -483,9 +479,9 @@ const newBrain = new Brainy({ storage: { type: 's3' } }) await oldBrain.init() await newBrain.init() -// Transfer all data -const data = await oldBrain.export() -await newBrain.import(data) +// Transfer all data via a portable backup +const data = await oldBrain.data().then(d => d.export(undefined, { includeVectors: true })) +await newBrain.data().then(d => d.import(data)) ``` ## Performance Tuning diff --git a/docs/augmentations/EXAMPLES.md b/docs/augmentations/EXAMPLES.md index 142f68fd..808139ba 100644 --- a/docs/augmentations/EXAMPLES.md +++ b/docs/augmentations/EXAMPLES.md @@ -546,11 +546,9 @@ const appComponent = await brain.findOne({ type: 'ReactComponent', name: 'App' } const imports = await brain.getRelated(appComponent.id, 'Imports') console.log(`App component imports:`, imports.map(c => c.name)) -// Export diagram -const diagram = await brain.export({ - format: 'react-diagram' -}) -console.log(diagram.diagram) // Mermaid diagram +// Export the analyzed component graph as a portable BackupData document +const graph = await brain.data().then(d => d.export({ type: 'ReactComponent' })) +console.log(`Exported ${graph.stats.entityCount} components, ${graph.stats.relationCount} edges`) ``` ### Step 4: Premium Licensing (Optional) diff --git a/docs/guides/export-and-import.md b/docs/guides/export-and-import.md new file mode 100644 index 00000000..3f367d33 --- /dev/null +++ b/docs/guides/export-and-import.md @@ -0,0 +1,173 @@ +--- +title: Export & Import (portable graph) +slug: guides/export-and-import +public: true +category: guides +template: guide +order: 9 +description: Export part or all of a brain to a portable, versioned BackupData document and import it back — by id, collection, connected neighbourhood, VFS subtree, predicate, or the whole brain. Covers vectors, edge policy, conflict handling, and id remapping. +next: + - guides/subtypes-and-facets + - api/README +--- + +# Export & Import (portable graph) + +`brain.data()` exposes a **portable graph export/import** API. One method serializes a +graph — an item, a collection, a connected neighbourhood, a VFS subtree, a predicate +match, or the whole brain — into a single versioned JSON document (`BackupData`); the +inverse restores it. + +```typescript +const data = await brain.data() + +const backup = await data.export() // whole brain → BackupData +await data.import(backup) // restore (merge by id, re-embed if no vectors) +``` + +It is **portable** (human-readable JSON), **versioned** (`formatVersion`, so a 7.x export +imports cleanly into 8.0), and **current-state** (the entities and edges as they are now — +no generation history). Use it for portable artifacts, partial backups, cross-environment +moves, and version upgrades. + +## When to use which + +| You want… | Use | +|-----------|-----| +| A portable, partial-or-whole, cross-version graph document | **`brain.data().export()` / `import()`** (this guide) | +| To ingest a CSV / PDF / Excel / JSON **file** as new entities | `brain.import(file)` — see [Import Anything](./import-anything.md) | + +The two are different operations that happen to share a verb: `import(file)` parses a +foreign document into new entities; `data().import(backup)` restores a graph this API +exported. + +## Exporting + +```typescript +export(selector?, options?): Promise +``` + +### Selectors — *what* to export + +Omit the selector to export the whole brain. Otherwise pick a node set: + +| Scenario | Selector | +|----------|----------| +| Just an item (or items) | `{ ids: ['a', 'b'] }` | +| A collection + its children | `{ collection: collectionId }` (alias `memberOf`) | +| A connected neighbourhood | `{ connected: { from: id, depth: 2, verbs?, direction? } }` | +| A VFS directory / file (+ subtree) | `{ vfsPath: '/docs', recursive?: true }` | +| Everything matching a predicate | `{ type, subtype, where, service }` | +| The whole brain | *(omit)* | + +The selector reuses `find()`'s grammar — *"export what `find()` would match, minus ranking +and limit."* Structural and predicate selectors **compose** — a structural selector picks +the nodes, and predicate keys then filter them: + +```typescript +// Members of a collection whose status is "open" +await data.export({ collection: collectionId, where: { status: 'open' } }) +``` + +If you already have results from `find()`, export exactly those with the `ids` selector: + +```typescript +const hits = await brain.find({ type: NounType.Document }) +await data.export({ ids: hits.map(r => r.id) }) +``` + +### Options — *how* to serialize + +| Option | Default | Effect | +|--------|---------|--------| +| `includeVectors` | `false` | Carry embedding vectors verbatim. Off ⇒ `import()` re-embeds from `data`. | +| `includeContent` | `false` | Include VFS file bytes in `blobs` so files round-trip byte-identically. | +| `includeSystem` | `false` | Include `system` entities such as the VFS root. | +| `edges` | `'induced'` | `'induced'` (both endpoints in the set), `'incident'` (also dangling edges, recorded in `danglingIds`), or `'none'` (nodes only). | + +```typescript +// A self-contained subgraph with vectors, ready to restore elsewhere +const subset = await data.export({ ids }, { includeVectors: true }) + +// A VFS subtree including file bytes +const tree = await data.export({ vfsPath: '/docs' }, { includeContent: true }) +``` + +## Importing + +```typescript +import(backup, options?): Promise +``` + +```typescript +const result = await brain.data().then(d => d.import(backup, { onConflict: 'merge' })) +// → { imported, merged, skipped, reembedded, blobsWritten, errors } +``` + +| Option | Default | Effect | +|--------|---------|--------| +| `onConflict` | `'merge'` | `'merge'` (update existing id in place — assemble many backups), `'replace'` (delete + recreate), or `'skip'`. | +| `reembed` | `'auto'` | `'auto'` (use the carried vector, else re-embed from `data`) or `'never'` (require a carried vector; record an error if absent). | +| `remapIds` | — | Rewrite every id on the way in, e.g. to clone a template subgraph under fresh ids. | + +The default `onConflict: 'merge'` is what lets you assemble one working graph from many +exported documents that share entity ids — re-importing an id merges rather than duplicates. + +```typescript +// Clone a subgraph under fresh ids (a copy, not a move) +const remap = new Map(backup.entities.map(e => [e.id, crypto.randomUUID()])) +await data.import(backup, { remapIds: id => remap.get(id) ?? id }) +``` + +## The `BackupData` format + +```jsonc +{ + "format": "brainy-backup", + "formatVersion": 1, // import gates on this (cross-version migration) + "brainyVersion": "7.32.0", + "createdAt": "2026-06-16T…Z", + "embedding": { "model": "all-MiniLM-L6-v2", "dimensions": 384 }, + "selector": { … }, // echoes what was exported (provenance) + "entities": [ + { + "id": "…", "type": "Document", "subtype": "invoice", + "data": "…", // the embedding source + "confidence": 1, "weight": 1, "service": "…", + "vector": [ … ], // only with includeVectors + "metadata": { … } // custom fields only (reserved fields are top-level) + } + ], + "relations": [ + { "id":"…", "from":"…", "to":"…", "type":"Contains", "subtype":"…", + "weight":1, "confidence":1, "metadata": { … } } + ], + "blobs": { "": "" }, // only with includeContent + "danglingIds": [ "…" ], // only with edges:'incident' + "stats": { "entityCount": 0, "relationCount": 0, "blobCount": 0, "vectorDimensions": 384 } +} +``` + +Standard fields (`subtype`, `data`, `confidence`, `weight`, `service`) sit at the top level +of each entity; `metadata` holds **only** custom user fields — mirroring the in-memory +`Entity` shape, so `import()` maps each field to its dedicated parameter. + +## Cross-version (7.x → 8.0) + +Because the document is shared and versioned, a backup written by 7.x imports into 8.0: +`formatVersion` is read forward, `subtype` is carried so 8.0 re-types correctly, and the +same 384-dimension model on both lines means `includeVectors:false` re-embeds identically +(or `true` carries vectors verbatim). The format is **current-state** — if you need a +whole-brain snapshot *with* generation history, that is a separate native facility +(`db.persist()` / `Brainy.load()` on 8.0). + +## VFS + +VFS directories are `Collection` entities and files are entities linked by `Contains`, so +the whole filesystem (or any subtree) exports through the `vfsPath` selector: + +```typescript +await data.export({ vfsPath: '/' }, { includeContent: true }) // all VFS + bytes +await data.export({ vfsPath: '/docs' }, { includeContent: true }) // one directory +await data.export({ vfsPath: '/a/b.txt' }, { includeContent: true }) // one file +``` diff --git a/docs/guides/framework-integration.md b/docs/guides/framework-integration.md index 23364178..fab81011 100644 --- a/docs/guides/framework-integration.md +++ b/docs/guides/framework-integration.md @@ -540,11 +540,11 @@ export async function generateStaticProps() { }) await brain.init() - // Build search index - const allContent = await brain.export() + // Build a search index — export the whole brain as a portable BackupData document + const backup = await brain.data().then(d => d.export()) return { - props: { searchIndex: allContent } + props: { searchIndex: backup.entities } } } ``` diff --git a/src/api/DataAPI.ts b/src/api/DataAPI.ts index 8ba1a49f..b873a9ed 100644 --- a/src/api/DataAPI.ts +++ b/src/api/DataAPI.ts @@ -1,302 +1,495 @@ /** - * Data Management API for Brainy 3.0 - * Provides backup, restore, import, export, and data management + * @module DataAPI + * @description Portable graph backup/restore for Brainy — the `BackupData v1` + * export/import format plus `clear()`/`getStats()` data-management helpers. + * + * Accessed via `brain.data()`. The two headline methods are: + * + * - **`export(selector?, options?) → BackupData`** — serialize a graph (an item, a + * collection + children, a connected neighbourhood, a VFS subtree, a predicate + * match, or the whole brain) into ONE versioned, portable JSON document. + * - **`import(backup, options?) → ImportResult`** — restore a `BackupData` into the + * brain (dedup-by-id merge by default), re-embedding from `data` when vectors are + * absent. + * + * This is the **portable** round-trip: human-readable, partial-or-whole, and + * cross-version (a `formatVersion: 1` document written by 7.x imports cleanly into + * 8.0). It is distinct from `brain.import()` (file ingestion — CSV/PDF/Excel/JSON) + * and, on 8.0, from `db.persist()`/`Brainy.load()` (the native whole-brain snapshot, + * which preserves generation history but is neither portable JSON nor cross-version). + * + * **Design decision (reserved-field split):** entities carry Brainy's standard fields + * (`subtype`, `data`, `confidence`, `weight`, `service`, `createdBy`, `createdAt`) at + * the TOP LEVEL of each `BackupEntity`, and `metadata` holds ONLY custom user fields — + * mirroring the in-memory `Entity` shape, so `import()` maps each field to its dedicated + * `add()`/`relate()` parameter rather than dumping everything into the metadata bag. */ -import { StorageAdapter, HNSWNoun, GraphVerb } from '../coreTypes.js' +import { StorageAdapter } from '../coreTypes.js' import { Entity, Relation } from '../types/brainy.types.js' import { NounType, VerbType } from '../types/graphTypes.js' +import { getBrainyVersion } from '../utils/version.js' -export interface BackupOptions { - includeVectors?: boolean - compress?: boolean - format?: 'json' | 'binary' -} +/** The fixed entity id of the VFS root collection (excluded from exports unless `includeSystem`). */ +const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' +/** Magic string identifying a Brainy portable backup document. */ +const BACKUP_FORMAT = 'brainy-backup' +/** Current portable-format version. Import gates on this for cross-version migration. */ +const BACKUP_FORMAT_VERSION = 1 +/** Default embedding model label (informational; `dimensions` is the real compat gate). */ +const DEFAULT_EMBED_MODEL = 'all-MiniLM-L6-v2' +/** Storage-layer fetch ceiling for enumerations (well above any single-brain entity count). */ +const ENUMERATION_LIMIT = 1_000_000 +/** Per-node relation fetch ceiling (filtered by source/target, so no unfiltered-scan warning). */ +const RELATION_FETCH_LIMIT = 100_000 -export interface RestoreOptions { - merge?: boolean - overwrite?: boolean - validate?: boolean -} - -export interface ImportOptions { - format: 'json' | 'csv' - mapping?: Record - batchSize?: number - validate?: boolean -} - -export interface ExportOptions { - format?: 'json' | 'csv' - filter?: { - type?: NounType | NounType[] - where?: Record - service?: string - } - includeVectors?: boolean -} - -export interface BackupData { - version: string - timestamp: number - entities: Array<{ - id: string - vector?: number[] - type: string - metadata: any - service?: string - }> - relations: Array<{ - id: string +/** + * @description Selects WHICH part of the graph to export. Reuses `find()`'s grammar + * (export "what find would match, minus ranking/limit") plus export-specific selectors. + * Omit the selector entirely (or pass `{}`) to export the whole brain. + * + * Structural selectors (`ids` / `collection` / `connected` / `vfsPath`) and predicate + * selectors (`type` / `subtype` / `where` / `service` / `visibility`) **compose**: a + * structural selector picks the node set, and any predicate keys then filter it + * (e.g. `{ collection: id, where: { status: 'open' } }` = members matching the predicate). + */ +export interface ExportSelector { + /** Exactly these entity ids. */ + ids?: string[] + /** A collection id → the collection + its transitive `Contains` members. */ + collection?: string + /** Alias for `collection`. */ + memberOf?: string + /** An entity + its N-hop neighbourhood (reuses graph traversal). */ + connected?: { + /** Start entity id. */ from: string - to: string - type: string - weight: number - metadata?: any - }> - config?: Record + /** Hops to traverse (default: 1). */ + depth?: number + /** Restrict to these verb types (default: all). */ + verbs?: VerbType[] + /** Edge direction to follow (default: `'out'`). */ + direction?: 'out' | 'in' | 'both' + } + /** A VFS path → the directory/file + (for a directory) its `Contains` subtree. */ + vfsPath?: string + /** For `vfsPath` directories: include the whole subtree (default: true). */ + recursive?: boolean + /** For `collection`/`vfsPath`: cap traversal depth (default: unbounded). */ + depth?: number + /** Predicate: entity type(s). */ + type?: NounType | NounType[] + /** Predicate: entity subtype(s). */ + subtype?: string | string[] + /** Predicate: exact-match metadata fields. */ + where?: Record + /** Predicate: multi-tenancy service id. */ + service?: string + /** Predicate: visibility (`'public'` matches entities with no explicit visibility). */ + visibility?: string +} + +/** + * @description Controls HOW the selected graph is serialized. + */ +export interface ExportOptions { + /** Include embedding vectors verbatim (default: false → `import()` re-embeds from `data`). */ + includeVectors?: boolean + /** Include VFS file bytes in `blobs` so files round-trip byte-identically (default: false). */ + includeContent?: boolean + /** Include `visibility:'system'` entities (e.g. the VFS root) (default: false). */ + includeSystem?: boolean + /** + * Which edges to include (default: `'induced'`): + * - `'induced'` — only edges whose BOTH endpoints are in the node set (self-contained subgraph). + * - `'incident'` — also edges that dangle to outside ids (recorded in `danglingIds`). + * - `'none'` — nodes only. + */ + edges?: 'induced' | 'incident' | 'none' +} + +/** + * @description Controls how a `BackupData` is applied to the brain on `import()`. + */ +export interface ImportOptions { + /** + * Conflict policy when an entity id already exists (default: `'merge'`): + * - `'merge'` — update in place (dedup-by-id; the default that lets you assemble many backups). + * - `'replace'` — delete then re-create. + * - `'skip'` — leave the existing entity untouched. + */ + onConflict?: 'merge' | 'replace' | 'skip' + /** + * Vector policy (default: `'auto'`): + * - `'auto'` — use the carried vector when present, otherwise re-embed from `data`. + * - `'never'` — use the carried vector when present, otherwise record an error (no re-embed). + */ + reembed?: 'auto' | 'never' + /** Rewrite every id on the way in (e.g. to clone a template subgraph under fresh ids). */ + remapIds?: (id: string) => string +} + +/** One entity in a `BackupData`. Standard fields top-level; `metadata` is custom-only. */ +export interface BackupEntity { + id: string + /** NounType value. */ + type: string + /** Per-product sub-classification. */ + subtype?: string + /** Visibility (omitted ⇒ `'public'`; never `'system'` unless `includeSystem`). */ + visibility?: string + /** Opaque content payload (the embedding source). */ + data?: any + /** Type-classification confidence (0–1). */ + confidence?: number + /** Entity importance/salience (0–1). */ + weight?: number + /** Multi-tenancy service id. */ + service?: string + /** Provenance: what created this entity. */ + createdBy?: any + /** Original creation timestamp (informational; the target brain assigns its own on import). */ + createdAt?: number + /** Embedding vector — present only when exported with `includeVectors`. */ + vector?: number[] + /** Custom user fields only (reserved fields live at the top level). */ + metadata?: any +} + +/** One relation (edge) in a `BackupData`. */ +export interface BackupRelation { + id: string + from: string + to: string + /** VerbType value. */ + type: string + /** Per-product edge sub-classification. */ + subtype?: string + /** Visibility (omitted ⇒ `'public'`). */ + visibility?: string + /** Connection strength (0–1). */ + weight?: number + /** Relationship certainty (0–1). */ + confidence?: number + /** Custom user fields on the edge. */ + metadata?: any +} + +/** + * @description A self-describing, versioned, portable graph document. The same shape + * is produced/consumed on 7.x and 8.0; `formatVersion` gates cross-version migration. + */ +export interface BackupData { + /** Always `'brainy-backup'` — identifies the document type. */ + format: typeof BACKUP_FORMAT + /** Integer format version (import gates on this). */ + formatVersion: number + /** The Brainy version that produced the document (informational). */ + brainyVersion: string + /** ISO-8601 creation time. */ + createdAt: string + /** Embedding manifest — `import()` verifies dimension compatibility before re-embedding. */ + embedding: { model: string; dimensions: number } + /** Echo of the selector that produced this document (provenance). */ + selector?: ExportSelector + /** Exported entities. */ + entities: BackupEntity[] + /** Exported relations. */ + relations: BackupRelation[] + /** VFS file bytes keyed by sha256 — present only with `includeContent`. */ + blobs?: Record + /** Endpoints referenced by `edges:'incident'` that fell outside the node set. */ + danglingIds?: string[] + /** Summary counts. */ stats: { entityCount: number relationCount: number + blobCount: number vectorDimensions?: number } } +/** Outcome of an `import()`. */ export interface ImportResult { - successful: number - failed: number - errors: Array<{ item: any; error: string }> - duration: number + /** New entities created. */ + imported: number + /** Existing entities merged (dedup-by-id). */ + merged: number + /** Existing entities left untouched (`onConflict:'skip'`). */ + skipped: number + /** Entities re-embedded from `data` because no vector was carried. */ + reembedded: number + /** VFS blob bytes written. */ + blobsWritten: number + /** Per-record failures (id + message); the import continues past individual errors. */ + errors: Array<{ id: string; error: string }> } +/** + * @description Data-management API for a Brainy instance: portable graph + * `export()`/`import()` plus `clear()`/`getStats()`. Constructed by `brain.data()`. + */ export class DataAPI { - private brain: any // Reference to Brainy instance for neural import - + /** + * @param storage - The brain's storage adapter (used for blob bytes + bulk enumeration). + * @param brain - The owning Brainy instance (drives export/import via its public API). + */ constructor( private storage: StorageAdapter, - private getEntity: (id: string) => Promise, - private getRelation?: (id: string) => Promise, - brain?: any - ) { - this.brain = brain + private brain: any + ) {} + + // ============================================================================ + // EXPORT + // ============================================================================ + + /** + * @description Serialize part or all of the graph into a portable `BackupData`. + * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}. + * @param options - HOW to export (vectors, file bytes, edge policy). See {@link ExportOptions}. + * @returns A versioned, portable `BackupData` document. + * @example + * // A single workbench's members (exact id set), with vectors: + * const backup = await brain.data().export({ ids }, { includeVectors: true }) + * @example + * // A collection and everything under it: + * const backup = await brain.data().export({ collection: collectionId }) + * @example + * // A VFS subtree including file bytes: + * const backup = await brain.data().export({ vfsPath: '/docs' }, { includeContent: true }) + * @example + * // The whole brain: + * const backup = await brain.data().export() + */ + async export(selector: ExportSelector = {}, options: ExportOptions = {}): Promise { + if (!this.brain) { + throw new Error('DataAPI.export() requires a Brainy instance (use brain.data().export()).') + } + const { + includeVectors = false, + includeContent = false, + includeSystem = false, + edges = 'induced' + } = options + + // 1. Resolve the candidate node set (structural selector, or all ids for whole/predicate). + let candidateIds = await this.resolveCandidates(selector, includeSystem) + + // 2. Read canonical entities (reserved fields top-level, metadata custom-only). + const entityMap = await this.brain.batchGet(candidateIds, { includeVectors }) + + // 3. Apply predicate filtering on the canonical entities (handles predicate-only + compose). + let idSet: Set + if (this.hasPredicate(selector)) { + idSet = new Set() + for (const id of candidateIds) { + const e = entityMap.get(id) + if (e && this.matchesPredicate(e, selector)) idSet.add(id) + } + } else { + idSet = new Set(candidateIds.filter((id: string) => entityMap.has(id))) + } + + // 4. Build entity records. + const entities: BackupEntity[] = [] + for (const id of idSet) { + const e = entityMap.get(id) + if (e) entities.push(this.toBackupEntity(e, includeVectors)) + } + + // 5. Collect edges per policy. + const { relations, danglingIds } = await this.collectEdges(idSet, edges) + + // 6. Collect VFS blob bytes (only when requested). + let blobs: Record | undefined + if (includeContent) { + blobs = await this.collectBlobs(entities, entityMap) + } + + // 7. Resolve the embedding dimension (from a carried vector, else the brain's dimension). + const dimensions = + entities.find((e) => e.vector && e.vector.length)?.vector?.length ?? + this.detectDimensions() ?? + 384 + + const blobCount = blobs ? Object.keys(blobs).length : 0 + + return { + format: BACKUP_FORMAT, + formatVersion: BACKUP_FORMAT_VERSION, + brainyVersion: getBrainyVersion(), + createdAt: new Date().toISOString(), + embedding: { model: this.detectModel(), dimensions }, + selector, + entities, + relations, + ...(blobs && blobCount > 0 ? { blobs } : {}), + ...(danglingIds && danglingIds.length > 0 ? { danglingIds } : {}), + stats: { + entityCount: entities.length, + relationCount: relations.length, + blobCount, + vectorDimensions: dimensions + } + } } - async clear(params: { - entities?: boolean - relations?: boolean - config?: boolean - } = {}): Promise { - const { entities = true, relations = true, config = false } = params + // ============================================================================ + // IMPORT + // ============================================================================ + + /** + * @description Restore a `BackupData` into the brain. Dedup-by-id merge by default, so + * assembling many backups that share entity ids merges rather than duplicates. Vectors + * are re-embedded from `data` when absent (`reembed:'auto'`). + * @param data - A `BackupData` document (must have `format:'brainy-backup'`). + * @param options - Conflict/vector/id-remap policy. See {@link ImportOptions}. + * @returns Counts of imported/merged/skipped/re-embedded entities + any per-record errors. + * @throws If `data` is not a `BackupData`, or its `formatVersion` is newer than supported. + * @example + * const result = await brain.data().import(backup, { onConflict: 'merge' }) + */ + async import(data: BackupData, options: ImportOptions = {}): Promise { + if (!this.brain) { + throw new Error('DataAPI.import() requires a Brainy instance (use brain.data().import()).') + } + if (!data || (data as any).format !== BACKUP_FORMAT) { + throw new Error( + `DataAPI.import() expects a BackupData document (format:'${BACKUP_FORMAT}'). ` + + `For file ingestion (CSV/PDF/Excel/JSON), use brain.import() instead.` + ) + } + if (typeof data.formatVersion === 'number' && data.formatVersion > BACKUP_FORMAT_VERSION) { + throw new Error( + `Backup formatVersion ${data.formatVersion} is newer than this Brainy supports ` + + `(max ${BACKUP_FORMAT_VERSION}). Upgrade Brainy to import this document.` + ) + } + + const { onConflict = 'merge', reembed = 'auto', remapIds } = options + const result: ImportResult = { + imported: 0, + merged: 0, + skipped: 0, + reembedded: 0, + blobsWritten: 0, + errors: [] + } + const mapId = (id: string) => (remapIds ? remapIds(id) : id) + + // 1. Write blob bytes first so file entities resolve their content. + if (data.blobs && Object.keys(data.blobs).length > 0) { + const blobStorage = (this.storage as any).blobStorage + if (blobStorage?.write) { + for (const [hash, b64] of Object.entries(data.blobs)) { + try { + await blobStorage.write(Buffer.from(b64, 'base64')) + result.blobsWritten++ + } catch (e) { + result.errors.push({ id: hash, error: `blob: ${(e as Error).message}` }) + } + } + } else { + for (const hash of Object.keys(data.blobs)) { + result.errors.push({ id: hash, error: 'blob: storage does not support binary blobs' }) + } + } + } + + // 2. Entities (so relation endpoints exist before edges are created). + for (const be of data.entities || []) { + const id = mapId(be.id) + try { + const exists = await this.entityExists(id) + if (exists) { + if (onConflict === 'skip') { + result.skipped++ + continue + } + if (onConflict === 'merge') { + await this.brain.update({ id, ...this.entityUpdateFields(be), merge: true }) + result.merged++ + continue + } + await this.brain.delete(id) // 'replace' + } + + const useVector = Array.isArray(be.vector) && be.vector.length > 0 + if (!useVector && reembed === 'never') { + result.errors.push({ id, error: 'no vector carried and reembed:never' }) + continue + } + await this.brain.add({ + id, + ...this.entityAddFields(be), + ...(useVector ? { vector: be.vector } : {}) + }) + result.imported++ + if (!useVector) result.reembedded++ + } catch (e) { + result.errors.push({ id, error: (e as Error).message }) + } + } + + // 3. Relations. + for (const br of data.relations || []) { + const from = mapId(br.from) + const to = mapId(br.to) + try { + await this.brain.relate({ + from, + to, + type: br.type as VerbType, + ...(br.subtype !== undefined ? { subtype: br.subtype } : {}), + ...(br.weight !== undefined ? { weight: br.weight } : {}), + ...(br.confidence !== undefined ? { confidence: br.confidence } : {}), + ...(br.metadata !== undefined ? { metadata: br.metadata } : {}) + }) + } catch (e) { + result.errors.push({ id: br.id, error: `relation: ${(e as Error).message}` }) + } + } + + return result + } + + // ============================================================================ + // CLEAR / STATS + // ============================================================================ + + /** + * @description Delete data from the brain. + * @param params - Which categories to clear (`entities`/`relations` default true; `config` false). + */ + async clear( + params: { entities?: boolean; relations?: boolean; config?: boolean } = {} + ): Promise { + const { entities = true, relations = true } = params if (entities) { - // Clear all entities - const nounsResult = await this.storage.getNouns({ - pagination: { limit: 1000000 } - }) - + const nounsResult = await this.storage.getNouns({ pagination: { limit: ENUMERATION_LIMIT } }) for (const noun of nounsResult.items) { await this.storage.deleteNoun(noun.id) } - - // Also clear the HNSW index if available if (this.brain?.index?.clear) { this.brain.index.clear() } - - // Clear metadata index if available if (this.brain?.metadataIndex) { - await this.brain.metadataIndex.rebuild() // Rebuild empty index + await this.brain.metadataIndex.rebuild() } } if (relations) { - // Clear all relations - const verbsResult = await this.storage.getVerbs({ - pagination: { limit: 1000000 } - }) - + const verbsResult = await this.storage.getVerbs({ pagination: { limit: ENUMERATION_LIMIT } }) for (const verb of verbsResult.items) { await this.storage.deleteVerb(verb.id) } } - - if (config) { - // Clear configuration would be handled by ConfigAPI - // For now, skip this - } } /** - * Import data from various formats - */ - async import(params: ImportOptions & { data: any }): Promise { - const { - data, - format, - mapping = {}, - batchSize = 100, - validate = true - } = params - - const result: ImportResult = { - successful: 0, - failed: 0, - errors: [], - duration: 0 - } - - const startTime = Date.now() - - try { - // ALWAYS use neural import for proper type matching - const { UniversalImportAPI } = await import('./UniversalImportAPI.js') - const universalImport = new UniversalImportAPI(this.brain) - await universalImport.init() - - // Convert to ImportSource format - const neuralResult = await universalImport.import({ - type: 'object', - data, - format: format || 'json', - metadata: { mapping, batchSize, validate } - }) - - // Convert neural result to ImportResult format - result.successful = neuralResult.stats.entitiesCreated - result.failed = 0 // Neural import always succeeds with best match - result.duration = neuralResult.stats.processingTimeMs - - // Log relationships created - if (neuralResult.stats.relationshipsCreated > 0) { - console.log(`Neural import also created ${neuralResult.stats.relationshipsCreated} relationships`) - } - - return result - } catch (error) { - // Fallback to legacy import ONLY if neural import fails to load - console.warn('Neural import failed, using legacy import:', error) - - let items: any[] = [] - - // Parse data based on format - switch (format) { - case 'json': - items = Array.isArray(data) ? data : [data] - break - - case 'csv': - // CSV parsing would go here - // For now, assume data is already parsed - items = data - break - - // Parquet format removed - not implemented - - default: - throw new Error(`Unsupported format: ${format}`) - } - - // Process items in batches - for (let i = 0; i < items.length; i += batchSize) { - const batch = items.slice(i, i + batchSize) - - for (const item of batch) { - try { - // Apply field mapping - const mapped = this.applyMapping(item, mapping) - - // Validate if requested - if (validate) { - this.validateImportItem(mapped) - } - - // Save entity - separate vector and metadata - const id = mapped.id || this.generateId() - const noun: HNSWNoun = { - id, - vector: mapped.vector || new Array(384).fill(0), - connections: new Map(), - level: 0 - } - - await this.storage.saveNoun(noun) - await this.storage.saveNounMetadata(id, { ...mapped, createdAt: Date.now() }) - result.successful++ - } catch (error) { - result.failed++ - result.errors.push({ - item, - error: (error as Error).message - }) - } - } - } - - result.duration = Date.now() - startTime - return result - } - } - - /** - * Export data to various formats - */ - async export(params: ExportOptions = {}): Promise { - const { - format = 'json', - filter = {}, - includeVectors = false - } = params - - // Get filtered entities - const nounsResult = await this.storage.getNouns({ - pagination: { limit: 1000000 } - }) - - let entities = nounsResult.items - - // Apply filters - if (filter.type) { - const types = Array.isArray(filter.type) ? filter.type : [filter.type] - entities = entities.filter(e => - types.includes(e.metadata?.noun as NounType) - ) - } - - if (filter.service) { - entities = entities.filter(e => - e.metadata?.service === filter.service - ) - } - - if (filter.where) { - entities = entities.filter(e => - this.matchesFilter(e.metadata, filter.where!) - ) - } - - // Format data based on export format - switch (format) { - case 'json': - return entities.map(e => ({ - id: e.id, - vector: includeVectors ? e.vector : undefined, - ...e.metadata - })) - - case 'csv': - // Convert to CSV format - // For now, return simplified format - return this.convertToCSV(entities) - - // Parquet format removed - not implemented - - default: - throw new Error(`Unsupported export format: ${format}`) - } - } - - /** - * Get storage statistics + * @description Summary counts for the brain. + * @returns Entity/relation totals and the vector dimensionality. */ async getStats(): Promise<{ entities: number @@ -304,14 +497,9 @@ export class DataAPI { storageSize?: number vectorDimensions?: number }> { - const nounsResult = await this.storage.getNouns({ - pagination: { limit: 1 } - }) - const verbsResult = await this.storage.getVerbs({ - pagination: { limit: 1 } - }) - - const firstNoun = nounsResult.items[0] + const nounsResult = await this.storage.getNouns({ pagination: { limit: 1 } }) + const verbsResult = await this.storage.getVerbs({ pagination: { limit: 1 } }) + const firstNoun = nounsResult.items[0] as any return { entities: nounsResult.totalCount || nounsResult.items.length, @@ -320,70 +508,342 @@ export class DataAPI { } } - // Helper methods + // ============================================================================ + // SELECTOR RESOLUTION (private) + // ============================================================================ - private applyMapping(item: any, mapping: Record): any { - const mapped: any = {} - - for (const [key, value] of Object.entries(item)) { - const mappedKey = mapping[key] || key - mapped[mappedKey] = value - } - - return mapped + /** True if the selector names a structural node set. */ + private hasStructural(s: ExportSelector): boolean { + return !!(s.ids || s.collection || s.memberOf || s.connected || s.vfsPath) } - private validateImportItem(item: any): void { - // Basic validation - if (!item || typeof item !== 'object') { - throw new Error('Invalid item: must be an object') - } - - // Could add more validation here + /** True if the selector carries predicate (filter) keys. */ + private hasPredicate(s: ExportSelector): boolean { + return ( + s.type !== undefined || + s.subtype !== undefined || + s.where !== undefined || + s.service !== undefined || + s.visibility !== undefined + ) } - private matchesFilter(metadata: any, filter: Record): boolean { - for (const [key, value] of Object.entries(filter)) { - if (metadata[key] !== value) { - return false + /** + * Resolve the candidate id list: the structural node set when a structural selector is + * present, otherwise every entity id (for predicate-only and whole-brain exports). + * Predicate filtering is applied later, on the canonical entities. + */ + private async resolveCandidates(s: ExportSelector, includeSystem: boolean): Promise { + let idSet: Set + + if (s.ids && s.ids.length) { + idSet = new Set(s.ids) + } else if (s.collection ?? s.memberOf) { + idSet = await this.resolveCollectionSubtree((s.collection ?? s.memberOf)!, s.depth) + } else if (s.connected) { + idSet = await this.resolveConnected(s.connected) + } else if (s.vfsPath) { + idSet = await this.resolveVfsPath(s.vfsPath, s.recursive ?? true, s.depth) + } else { + idSet = await this.allEntityIds() + } + + if (!includeSystem) idSet.delete(VFS_ROOT_ID) + return Array.from(idSet) + } + + /** Every entity id in the brain (storage-layer enumeration; no query-limit enforcement). */ + private async allEntityIds(): Promise> { + const result = await this.storage.getNouns({ pagination: { limit: ENUMERATION_LIMIT } }) + return new Set(result.items.map((n: any) => n.id)) + } + + /** A collection id + its transitive `Contains` members (BFS, optionally depth-capped). */ + private async resolveCollectionSubtree(rootId: string, depth?: number): Promise> { + const set = new Set([rootId]) + const maxDepth = depth ?? Infinity + let frontier = [rootId] + let d = 0 + while (frontier.length && d < maxDepth) { + const next: string[] = [] + for (const id of frontier) { + const rels = await this.brain.getRelations({ + from: id, + type: VerbType.Contains, + limit: RELATION_FETCH_LIMIT + }) + for (const r of rels as Relation[]) { + if (!set.has(r.to)) { + set.add(r.to) + next.push(r.to) + } + } } + frontier = next + d++ + } + return set + } + + /** An entity + its N-hop neighbourhood, following `verbs`/`direction`. */ + private async resolveConnected(c: NonNullable): Promise> { + const { from, depth = 1, verbs, direction = 'out' } = c + const set = new Set([from]) + let frontier = [from] + for (let d = 0; d < depth; d++) { + const next: string[] = [] + for (const id of frontier) { + const neighbours = await this.neighboursOf(id, direction, verbs) + for (const n of neighbours) { + if (!set.has(n)) { + set.add(n) + next.push(n) + } + } + } + frontier = next + if (!next.length) break + } + return set + } + + /** Neighbour ids of an entity in the requested direction, optionally verb-filtered. */ + private async neighboursOf( + id: string, + direction: 'out' | 'in' | 'both', + verbs?: VerbType[] + ): Promise { + const out: string[] = [] + if (direction === 'out' || direction === 'both') { + const rels = (await this.brain.getRelations({ from: id, limit: RELATION_FETCH_LIMIT })) as Relation[] + for (const r of rels) if (!verbs || verbs.includes(r.type)) out.push(r.to) + } + if (direction === 'in' || direction === 'both') { + const rels = (await this.brain.getRelations({ to: id, limit: RELATION_FETCH_LIMIT })) as Relation[] + for (const r of rels) if (!verbs || verbs.includes(r.type)) out.push(r.from) + } + return out + } + + /** A VFS path → the resolved entity, plus (for a directory) its subtree. */ + private async resolveVfsPath(path: string, recursive: boolean, depth?: number): Promise> { + const dirId = await this.resolveVfsPathToId(path) + if (!dirId) return new Set() + if (!recursive) { + const set = new Set([dirId]) + const rels = (await this.brain.getRelations({ + from: dirId, + type: VerbType.Contains, + limit: RELATION_FETCH_LIMIT + })) as Relation[] + for (const r of rels) set.add(r.to) + return set + } + return this.resolveCollectionSubtree(dirId, depth) + } + + /** Walk `Contains` from the VFS root, matching each path segment against `metadata.name`. */ + private async resolveVfsPathToId(path: string): Promise { + const segments = path.split('/').filter(Boolean) + let currentId = VFS_ROOT_ID + for (const seg of segments) { + const rels = (await this.brain.getRelations({ + from: currentId, + type: VerbType.Contains, + limit: RELATION_FETCH_LIMIT + })) as Relation[] + let found: string | null = null + for (const r of rels) { + const child = await this.brain.get(r.to) + if (child?.metadata?.name === seg) { + found = r.to + break + } + } + if (!found) return null + currentId = found + } + return currentId + } + + /** True if an entity satisfies the selector's predicate keys. */ + private matchesPredicate(e: Entity, s: ExportSelector): boolean { + if (s.type !== undefined) { + const types = Array.isArray(s.type) ? s.type : [s.type] + if (!types.includes(e.type)) return false + } + if (s.subtype !== undefined) { + const subs = Array.isArray(s.subtype) ? s.subtype : [s.subtype] + if (e.subtype === undefined || !subs.includes(e.subtype)) return false + } + if (s.service !== undefined && e.service !== s.service) return false + if (s.visibility !== undefined) { + const vis = (e as any).visibility ?? 'public' + if (vis !== s.visibility) return false + } + if (s.where && !this.matchesWhere(e.metadata, s.where)) return false + return true + } + + /** Exact-match metadata predicate. */ + private matchesWhere(metadata: any, where: Record): boolean { + if (!metadata) return false + for (const [key, value] of Object.entries(where)) { + if (metadata[key] !== value) return false } return true } - private convertToCSV(entities: Array<{ id: string; metadata?: any }>): string { - if (entities.length === 0) return '' + // ============================================================================ + // SERIALIZATION HELPERS (private) + // ============================================================================ - // Get all unique keys from metadata - const keys = new Set() - for (const entity of entities) { - if (entity.metadata) { - Object.keys(entity.metadata).forEach(k => keys.add(k)) + /** Map a canonical `Entity` to a `BackupEntity` (reserved fields top-level). */ + private toBackupEntity(e: Entity, includeVectors: boolean): BackupEntity { + const be: BackupEntity = { id: e.id, type: e.type as string } + if (e.subtype !== undefined) be.subtype = e.subtype + const vis = (e as any).visibility + if (vis !== undefined && vis !== 'public') be.visibility = vis + if (e.data !== undefined) be.data = e.data + if (e.confidence !== undefined) be.confidence = e.confidence + if (e.weight !== undefined) be.weight = e.weight + if (e.service !== undefined) be.service = e.service + if (e.createdBy !== undefined) be.createdBy = e.createdBy + if (e.createdAt !== undefined) be.createdAt = e.createdAt + if (includeVectors && e.vector && e.vector.length) be.vector = e.vector + if (e.metadata && Object.keys(e.metadata).length) be.metadata = e.metadata + return be + } + + /** Map a canonical `Relation` to a `BackupRelation`. */ + private toBackupRelation(r: Relation): BackupRelation { + const br: BackupRelation = { id: r.id, from: r.from, to: r.to, type: r.type as string } + if (r.subtype !== undefined) br.subtype = r.subtype + const vis = (r as any).visibility + if (vis !== undefined && vis !== 'public') br.visibility = vis + if (r.weight !== undefined) br.weight = r.weight + if (r.confidence !== undefined) br.confidence = r.confidence + if (r.metadata && Object.keys(r.metadata).length) br.metadata = r.metadata + return br + } + + /** Collect edges among the node set per the edge policy. */ + private async collectEdges( + idSet: Set, + edges: 'induced' | 'incident' | 'none' + ): Promise<{ relations: BackupRelation[]; danglingIds?: string[] }> { + if (edges === 'none') return { relations: [] } + + const relations: BackupRelation[] = [] + const dangling = new Set() + const seen = new Set() + + // Outgoing edges from each in-set node (captures every induced edge exactly once). + for (const id of idSet) { + const rels = (await this.brain.getRelations({ from: id, limit: RELATION_FETCH_LIMIT })) as Relation[] + for (const r of rels) { + if (seen.has(r.id)) continue + const toIn = idSet.has(r.to) + if (edges === 'induced' && !toIn) continue + if (!toIn) dangling.add(r.to) + seen.add(r.id) + relations.push(this.toBackupRelation(r)) } } - // Create CSV header - const headers = ['id', ...Array.from(keys)] - const rows = [headers.join(',')] - - // Add data rows - for (const entity of entities) { - const row = [entity.id] - for (const key of keys) { - const value = entity.metadata?.[key] || '' - // Escape values that contain commas - const escaped = String(value).includes(',') - ? `"${String(value).replace(/"/g, '""')}"` - : String(value) - row.push(escaped) + // For 'incident', also capture edges arriving from outside the set. + if (edges === 'incident') { + for (const id of idSet) { + const rels = (await this.brain.getRelations({ to: id, limit: RELATION_FETCH_LIMIT })) as Relation[] + for (const r of rels) { + if (seen.has(r.id)) continue + if (!idSet.has(r.from)) { + dangling.add(r.from) + seen.add(r.id) + relations.push(this.toBackupRelation(r)) + } + } } - rows.push(row.join(',')) } - return rows.join('\n') + return dangling.size > 0 ? { relations, danglingIds: Array.from(dangling) } : { relations } } - private generateId(): string { - return `import_${Date.now()}_${Math.random().toString(36).substr(2, 9)}` + /** Read VFS file bytes (base64) for file entities, keyed by content hash. */ + private async collectBlobs( + entities: BackupEntity[], + entityMap: Map + ): Promise> { + const blobs: Record = {} + const blobStorage = (this.storage as any).blobStorage + if (!blobStorage?.read) return blobs + + for (const be of entities) { + const e = entityMap.get(be.id) as any + const storageMeta = e?.metadata?.storage + const hash = storageMeta?.hash + if (storageMeta?.type === 'blob' && hash && !blobs[hash]) { + try { + const buf = await blobStorage.read(hash) + blobs[hash] = Buffer.from(buf).toString('base64') + } catch { + // Referenced blob bytes are unreadable (storage drift): skip — the file entity's + // structure still travels, and stats.blobCount reflects what was actually captured. + } + } + } + return blobs } -} \ No newline at end of file + + /** Best-effort embedding model label from the brain's configuration. */ + private detectModel(): string { + return this.brain?.config?.embedding?.model || DEFAULT_EMBED_MODEL + } + + /** Embedding dimensionality from the brain's configuration, if exposed. */ + private detectDimensions(): number | undefined { + const dim = this.brain?.config?.dimensions ?? this.brain?.dimensions + return typeof dim === 'number' ? dim : undefined + } + + // ============================================================================ + // IMPORT HELPERS (private) + // ============================================================================ + + /** Existence check that never throws on id-format quirks (storage-layer read). */ + private async entityExists(id: string): Promise { + try { + const meta = await this.storage.getNounMetadata(id) + return !!meta + } catch { + return false + } + } + + /** `add()` params from a `BackupEntity` (excludes brain-managed fields like `createdAt`). */ + private entityAddFields(be: BackupEntity): Record { + const fields: Record = { + data: be.data, + type: be.type as NounType + } + if (be.subtype !== undefined) fields.subtype = be.subtype + if (be.service !== undefined) fields.service = be.service + if (be.confidence !== undefined) fields.confidence = be.confidence + if (be.weight !== undefined) fields.weight = be.weight + if (be.metadata !== undefined) fields.metadata = be.metadata + return fields + } + + /** `update()` params from a `BackupEntity` (for `onConflict:'merge'`). */ + private entityUpdateFields(be: BackupEntity): Record { + const fields: Record = {} + if (be.data !== undefined) fields.data = be.data + if (be.type !== undefined) fields.type = be.type as NounType + if (be.subtype !== undefined) fields.subtype = be.subtype + if (be.confidence !== undefined) fields.confidence = be.confidence + if (be.weight !== undefined) fields.weight = be.weight + if (be.metadata !== undefined) fields.metadata = be.metadata + if (Array.isArray(be.vector) && be.vector.length) fields.vector = be.vector + return fields + } +} diff --git a/src/brainy.ts b/src/brainy.ts index a8c6e8e4..33bcb91d 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -5770,12 +5770,7 @@ export class Brainy implements BrainyInterface { */ async data() { const { DataAPI } = await import('./api/DataAPI.js') - return new DataAPI( - this.storage, - (id: string) => this.get(id), - undefined, // No getRelation method yet - this - ) + return new DataAPI(this.storage, this) } /** diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts index c8509f5e..e0322bfe 100644 --- a/src/cli/commands/core.ts +++ b/src/cli/commands/core.ts @@ -876,60 +876,66 @@ export const coreCommands = { const brain = getBrainy() const format = options.format || 'json' - // Export all data + // Export the whole brain as a portable BackupData document (vectors + VFS file bytes). const dataApi = await brain.data() - const data = await dataApi.export({ format: 'json' }) + const backup = await dataApi.export(undefined, { + includeVectors: true, + includeContent: true + }) let output = '' - + + const csvCell = (v: any): string => { + if (v === undefined || v === null) return '' + const s = typeof v === 'object' ? JSON.stringify(v) : String(v) + return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s + } + switch (format) { case 'json': - output = options.pretty - ? JSON.stringify(data, null, 2) - : JSON.stringify(data) + output = options.pretty + ? JSON.stringify(backup, null, 2) + : JSON.stringify(backup) break - - case 'jsonl': - if (Array.isArray(data)) { - output = data.map(item => JSON.stringify(item)).join('\n') - } else { - output = JSON.stringify(data) - } + + case 'jsonl': { + // NDJSON: a header line, then one line per entity, then one line per relation. + const { entities, relations, ...header } = backup + const lines: string[] = [JSON.stringify({ kind: 'header', ...header })] + for (const e of entities) lines.push(JSON.stringify({ kind: 'entity', ...e })) + for (const r of relations) lines.push(JSON.stringify({ kind: 'relation', ...r })) + output = lines.join('\n') break - - case 'csv': - if (Array.isArray(data) && data.length > 0) { - // Get all unique keys for headers + } + + case 'csv': { + // CSV represents entities only — a graph's edges and blobs can't be tabular. + const entities = backup.entities + if (entities.length > 0) { const headers = new Set() - data.forEach(item => { - Object.keys(item).forEach(key => headers.add(key)) - }) + entities.forEach((e: Record) => Object.keys(e).forEach(k => headers.add(k))) const headerArray = Array.from(headers) - - // Create CSV output = headerArray.join(',') + '\n' - output += data.map(item => { - return headerArray.map(h => { - const value = item[h] - if (typeof value === 'object') { - return JSON.stringify(value) - } - return value || '' - }).join(',') - }).join('\n') + output += entities + .map((e: Record) => headerArray.map(h => csvCell(e[h])).join(',')) + .join('\n') } break + } } - + if (file) { writeFileSync(file, output) spinner.succeed(`Exported to ${file}`) - + if (!options.json) { console.log(chalk.green(`✓ Successfully exported database to ${file}`)) console.log(chalk.dim(` Format: ${format}`)) - console.log(chalk.dim(` Items: ${Array.isArray(data) ? data.length : 1}`)) + console.log(chalk.dim(` Entities: ${backup.stats.entityCount} Relations: ${backup.stats.relationCount}`)) } else { - formatOutput({ file, format, count: Array.isArray(data) ? data.length : 1 }, options) + formatOutput( + { file, format, entityCount: backup.stats.entityCount, relationCount: backup.stats.relationCount }, + options + ) } } else { spinner.succeed('Export complete') diff --git a/src/index.ts b/src/index.ts index bacd8c5b..47e1b30b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -116,6 +116,18 @@ export { // Export version utilities export { getBrainyVersion } from './utils/version.js' +// Export portable graph backup/restore API (brain.data()) +export { DataAPI } from './api/DataAPI.js' +export type { + BackupData, + BackupEntity, + BackupRelation, + ExportSelector, + ExportOptions, + ImportOptions, + ImportResult +} from './api/DataAPI.js' + // Export plugin system export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js' export { PluginRegistry } from './plugin.js' diff --git a/tests/unit/api/data-backup.test.ts b/tests/unit/api/data-backup.test.ts new file mode 100644 index 00000000..2d3e88d1 --- /dev/null +++ b/tests/unit/api/data-backup.test.ts @@ -0,0 +1,311 @@ +/** + * Unit tests for the portable graph backup/restore API (brain.data()). + * + * Exercises BackupData v1 export()/import(): real round-trips against in-memory + * storage (no mocks of the API under test) — selectors, edge policy, vectors, + * conflict handling, id remapping, subtype fidelity, and cross-version guards. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { randomUUID } from 'node:crypto' +import { Brainy } from '../../../src/brainy' +import { createTestConfig } from '../../helpers/test-factory' +import { NounType, VerbType } from '../../../src/types/graphTypes' +import type { BackupData } from '../../../src/api/DataAPI' + +const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + +describe('DataAPI — portable graph backup/restore (BackupData v1)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + describe('format + whole-brain round-trip', () => { + it('exports a self-describing, versioned BackupData document', async () => { + const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' }) + const b = await brain.add({ data: 'Acme', type: NounType.Organization }) + await brain.relate({ from: a, to: b, type: VerbType.WorksWith, subtype: 'full-time' }) + + const backup = await brain.data().then((d) => d.export()) + + expect(backup.format).toBe('brainy-backup') + expect(backup.formatVersion).toBe(1) + expect(typeof backup.brainyVersion).toBe('string') + expect(backup.brainyVersion.length).toBeGreaterThan(0) + expect(typeof backup.createdAt).toBe('string') + expect(backup.embedding.dimensions).toBeGreaterThan(0) + expect(backup.stats.entityCount).toBe(backup.entities.length) + expect(backup.stats.relationCount).toBe(backup.relations.length) + // Two user entities, one relation (VFS root excluded by default). + expect(backup.entities.map((e) => e.id).sort()).toEqual([a, b].sort()) + expect(backup.relations).toHaveLength(1) + }) + + it('round-trips entities and relations through clear() + import()', async () => { + const a = await brain.add({ data: 'Alice', type: NounType.Person }) + const b = await brain.add({ data: 'Bob', type: NounType.Person }) + const c = await brain.add({ data: 'Carol', type: NounType.Person }) + await brain.relate({ from: a, to: b, type: VerbType.FriendOf }) + await brain.relate({ from: b, to: c, type: VerbType.FriendOf }) + + const backup = await brain.data().then((d) => d.export({}, { includeVectors: true })) + + await brain.data().then((d) => d.clear()) + const afterClear = await brain.find({ limit: 1000 }) + expect(afterClear.filter((r) => [a, b, c].includes(r.id))).toHaveLength(0) + + const result = await brain.data().then((d) => d.import(backup)) + expect(result.imported).toBe(3) + expect(result.errors).toHaveLength(0) + + const ra = await brain.get(a) + expect(ra?.id).toBe(a) + const rels = await brain.getRelations({ from: a }) + expect(rels.some((r) => r.to === b && r.type === VerbType.FriendOf)).toBe(true) + }) + + it('preserves subtype on both entities and relations', async () => { + const a = await brain.add({ data: 'Doc', type: NounType.Document, subtype: 'invoice' }) + const b = await brain.add({ data: 'Doc2', type: NounType.Document, subtype: 'receipt' }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'supersedes' }) + + const backup = await brain.data().then((d) => d.export()) + + const ea = backup.entities.find((e) => e.id === a) + expect(ea?.subtype).toBe('invoice') + expect(backup.relations[0].subtype).toBe('supersedes') + }) + }) + + describe('selectors', () => { + it('ids — exports exactly the requested entities', async () => { + const a = await brain.add({ data: 'A', type: NounType.Thing }) + const b = await brain.add({ data: 'B', type: NounType.Thing }) + await brain.add({ data: 'C', type: NounType.Thing }) + + const backup = await brain.data().then((d) => d.export({ ids: [a, b] })) + expect(backup.entities.map((e) => e.id).sort()).toEqual([a, b].sort()) + }) + + it('collection — exports the collection plus its transitive Contains members', async () => { + const root = await brain.add({ data: 'Folder', type: NounType.Collection }) + const child1 = await brain.add({ data: 'Child1', type: NounType.Document }) + const child2 = await brain.add({ data: 'Child2', type: NounType.Document }) + const grandchild = await brain.add({ data: 'Grandchild', type: NounType.Document }) + await brain.add({ data: 'Outside', type: NounType.Document }) + await brain.relate({ from: root, to: child1, type: VerbType.Contains }) + await brain.relate({ from: root, to: child2, type: VerbType.Contains }) + await brain.relate({ from: child1, to: grandchild, type: VerbType.Contains }) + + const backup = await brain.data().then((d) => d.export({ collection: root })) + expect(backup.entities.map((e) => e.id).sort()).toEqual( + [root, child1, child2, grandchild].sort() + ) + }) + + it('connected — exports an N-hop neighbourhood', async () => { + const a = await brain.add({ data: 'A', type: NounType.Thing }) + const b = await brain.add({ data: 'B', type: NounType.Thing }) + const c = await brain.add({ data: 'C', type: NounType.Thing }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + await brain.relate({ from: b, to: c, type: VerbType.RelatedTo }) + + const depth1 = await brain.data().then((d) => d.export({ connected: { from: a, depth: 1 } })) + expect(depth1.entities.map((e) => e.id).sort()).toEqual([a, b].sort()) + + const depth2 = await brain.data().then((d) => d.export({ connected: { from: a, depth: 2 } })) + expect(depth2.entities.map((e) => e.id).sort()).toEqual([a, b, c].sort()) + }) + + it('predicate — filters by type', async () => { + await brain.add({ data: 'P', type: NounType.Person }) + await brain.add({ data: 'D', type: NounType.Document }) + const backup = await brain.data().then((d) => d.export({ type: NounType.Person })) + expect(backup.entities.every((e) => e.type === NounType.Person)).toBe(true) + expect(backup.entities).toHaveLength(1) + }) + + it('compose — structural selector + predicate filter', async () => { + const root = await brain.add({ data: 'Folder', type: NounType.Collection }) + const open = await brain.add({ + data: 'Open', + type: NounType.Document, + metadata: { status: 'open' } + }) + const closed = await brain.add({ + data: 'Closed', + type: NounType.Document, + metadata: { status: 'closed' } + }) + await brain.relate({ from: root, to: open, type: VerbType.Contains }) + await brain.relate({ from: root, to: closed, type: VerbType.Contains }) + + const backup = await brain + .data() + .then((d) => d.export({ collection: root, where: { status: 'open' } })) + expect(backup.entities.map((e) => e.id)).toContain(open) + expect(backup.entities.map((e) => e.id)).not.toContain(closed) + }) + }) + + describe('edge policy', () => { + let a: string + let b: string + let c: string + + beforeEach(async () => { + a = await brain.add({ data: 'A', type: NounType.Thing }) + b = await brain.add({ data: 'B', type: NounType.Thing }) + c = await brain.add({ data: 'C', type: NounType.Thing }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) // both in {a,b} + await brain.relate({ from: b, to: c, type: VerbType.RelatedTo }) // dangles to c + }) + + it('induced (default) — only edges with both endpoints in the set', async () => { + const backup = await brain.data().then((d) => d.export({ ids: [a, b] })) + expect(backup.relations).toHaveLength(1) + expect(backup.relations[0].from).toBe(a) + expect(backup.danglingIds).toBeUndefined() + }) + + it('incident — includes dangling edges + records danglingIds', async () => { + const backup = await brain.data().then((d) => d.export({ ids: [a, b] }, { edges: 'incident' })) + expect(backup.relations).toHaveLength(2) + expect(backup.danglingIds).toContain(c) + }) + + it('none — nodes only', async () => { + const backup = await brain.data().then((d) => d.export({ ids: [a, b] }, { edges: 'none' })) + expect(backup.relations).toHaveLength(0) + }) + }) + + describe('vectors + re-embedding', () => { + it('omits vectors by default and re-embeds on import', async () => { + const a = await brain.add({ data: 'Re-embed me', type: NounType.Thing }) + const backup = await brain.data().then((d) => d.export({ ids: [a] })) + expect(backup.entities[0].vector).toBeUndefined() + + await brain.data().then((d) => d.clear()) + const result = await brain.data().then((d) => d.import(backup)) + expect(result.reembedded).toBe(1) + + const restored = await brain.get(a, { includeVectors: true }) + expect(restored?.vector?.length).toBeGreaterThan(0) + }) + + it('carries vectors verbatim with includeVectors (no re-embed)', async () => { + const a = await brain.add({ data: 'Keep my vector', type: NounType.Thing }) + const original = await brain.get(a, { includeVectors: true }) + + const backup = await brain.data().then((d) => d.export({ ids: [a] }, { includeVectors: true })) + expect(backup.entities[0].vector?.length).toBe(original?.vector?.length) + + await brain.data().then((d) => d.clear()) + const result = await brain.data().then((d) => d.import(backup)) + expect(result.reembedded).toBe(0) + + const restored = await brain.get(a, { includeVectors: true }) + expect(restored?.vector).toEqual(original?.vector) + }) + + it('reembed:never records an error when no vector is carried', async () => { + const a = await brain.add({ data: 'No vector', type: NounType.Thing }) + const backup = await brain.data().then((d) => d.export({ ids: [a] })) // no vectors + await brain.data().then((d) => d.clear()) + + const result = await brain.data().then((d) => d.import(backup, { reembed: 'never' })) + expect(result.imported).toBe(0) + expect(result.errors).toHaveLength(1) + expect(result.errors[0].error).toMatch(/reembed:never/) + }) + }) + + describe('conflict handling', () => { + it('merge (default) — updates existing entities in place', async () => { + const a = await brain.add({ data: 'V1', type: NounType.Thing, metadata: { n: 1 } }) + const backup = await brain.data().then((d) => d.export({ ids: [a] }, { includeVectors: true })) + + await brain.update({ id: a, metadata: { n: 99 }, merge: false }) + const result = await brain.data().then((d) => d.import(backup, { onConflict: 'merge' })) + expect(result.merged).toBe(1) + expect(result.imported).toBe(0) + + const restored = await brain.get(a) + expect(restored?.metadata?.n).toBe(1) + }) + + it('skip — leaves existing entities untouched', async () => { + const a = await brain.add({ data: 'Orig', type: NounType.Thing, metadata: { n: 1 } }) + const backup = await brain.data().then((d) => d.export({ ids: [a] })) + await brain.update({ id: a, metadata: { n: 2 }, merge: false }) + + const result = await brain.data().then((d) => d.import(backup, { onConflict: 'skip' })) + expect(result.skipped).toBe(1) + const restored = await brain.get(a) + expect(restored?.metadata?.n).toBe(2) + }) + }) + + describe('id remapping (clone)', () => { + it('imports a subgraph under fresh ids', async () => { + const a = await brain.add({ data: 'A', type: NounType.Thing }) + const b = await brain.add({ data: 'B', type: NounType.Thing }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo }) + const backup = await brain.data().then((d) => d.export({ ids: [a, b] }, { includeVectors: true })) + + const remap = new Map([ + [a, randomUUID()], + [b, randomUUID()] + ]) + const result = await brain + .data() + .then((d) => d.import(backup, { remapIds: (id) => remap.get(id) ?? id })) + expect(result.imported).toBe(2) + + const clonedA = await brain.get(remap.get(a)!) + expect(clonedA?.id).toBe(remap.get(a)) + // The original entities still exist (a clone, not a move). + expect((await brain.get(a))?.id).toBe(a) + const clonedRels = await brain.getRelations({ from: remap.get(a)! }) + expect(clonedRels.some((r) => r.to === remap.get(b))).toBe(true) + }) + }) + + describe('system entities', () => { + it('excludes the VFS root from a whole-brain export by default', async () => { + await brain.add({ data: 'User entity', type: NounType.Thing }) + const backup = await brain.data().then((d) => d.export()) + expect(backup.entities.map((e) => e.id)).not.toContain(VFS_ROOT_ID) + }) + }) + + describe('import validation', () => { + it('rejects a non-BackupData payload', async () => { + const d = await brain.data() + await expect(d.import({ entities: [] } as any)).rejects.toThrow(/BackupData/) + }) + + it('rejects a newer formatVersion', async () => { + const d = await brain.data() + const future: BackupData = { + format: 'brainy-backup', + formatVersion: 999, + brainyVersion: 'x', + createdAt: new Date().toISOString(), + embedding: { model: 'm', dimensions: 384 }, + entities: [], + relations: [], + stats: { entityCount: 0, relationCount: 0, blobCount: 0 } + } + await expect(d.import(future)).rejects.toThrow(/formatVersion/) + }) + }) +}) From adec0ba3c3a40aabca3c010c849f1dde21f4c59d Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 16 Jun 2026 15:58:47 -0700 Subject: [PATCH 014/179] chore(release): 7.32.0 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4eeee7c1..cdef7efd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.32.0](https://github.com/soulcraftlabs/brainy/compare/v7.31.8...v7.32.0) (2026-06-16) + +- feat: portable graph export()/import() (BackupData v1) on brain.data() (a408d37) + + ### [7.31.8](https://github.com/soulcraftlabs/brainy/compare/v7.31.7...v7.31.8) (2026-06-16) - fix: query-cap memory misread (MemAvailable + floor) + rootDirectory getter for native mmap fast-path (3f8e097) diff --git a/package-lock.json b/package-lock.json index c479349a..9909451c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.31.8", + "version": "7.32.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.31.8", + "version": "7.32.0", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index 31ad3502..7ac5af92 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.31.8", + "version": "7.32.0", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From edff637bfa08df292e4464ae175797912555b52f Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 17 Jun 2026 14:01:33 -0700 Subject: [PATCH 015/179] fix: getNouns().totalCount reports true total, not page size; quiet benign mmap-vector log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getNounsWithPagination returned collectedNouns.length as totalCount, but the type-first shard scan early-terminates at offset+limit — so getNouns({ pagination: { limit: 1 } }).totalCount was 1 for any non-empty brain. The index-rebuild gate calls exactly that, so cold starts logged "Small dataset (1 items) - rebuilding all indexes" and rebuilt from scratch regardless of corpus size (a production deployment saw this for an ~8,800-entity brain). Now reports the authoritative O(1) noun counter (maintained on add/delete, rehydrated from counts.json on init) as the unfiltered total and derives hasMore from it. Filtered scans unchanged. Layout-independent (branch/COW included). Also downgrade the "mmap-vector backend not wired" console.log to prodLog.debug: it is benign in the native-vector-index model (the native provider owns its own vector storage and has no setVectorBackend hook), but it fired on every init and was repeatedly mistaken for the cold-start cause. Regression: tests/unit/storage/getNouns-totalCount.test.ts. Full unit suite green (1505). --- RELEASES.md | 38 ++++++++ src/brainy.ts | 17 ++-- src/storage/baseStorage.ts | 21 ++++- .../unit/storage/getNouns-totalCount.test.ts | 88 +++++++++++++++++++ 4 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 tests/unit/storage/getNouns-totalCount.test.ts diff --git a/RELEASES.md b/RELEASES.md index ea16c8d2..72cb7d48 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,44 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so --- +## v7.32.1 — 2026-06-17 + +**Affected products:** consumers on `filesystem` / `mmap-filesystem` storage whose logs showed +`🔄 Small dataset (1 items) - rebuilding all indexes…` on cold start, or noise from a +`mmap-vector backend not wired` line on every init. Two fixes. Drop-in; no API or data changes. + +### Fix — `getNouns().totalCount` reports the true total, not the page size + +`storage.getNouns({ pagination: { limit } })` returned `totalCount` equal to the **page size**, not +the dataset total: the type-first shard scan early-terminates at `offset + limit` for memory +efficiency, and the page-collected length was returned as the total. So +`getNouns({ pagination: { limit: 1 } })` reported `totalCount: 1` for **any** non-empty brain. + +The index-rebuild gate uses exactly this call to size the corpus, so a cold start that needs a +rebuild logged `Small dataset (1 items) - rebuilding all indexes…` regardless of the real entity +count (a production deployment saw this for an ~8,800-entity brain — the rebuild then ran from +scratch instead of loading the persisted vector snapshot). + +`getNounsWithPagination` now reports the authoritative O(1) noun counter (maintained on every +add/delete and rehydrated from `counts.json` on init) as the unfiltered `totalCount`, and computes +`hasMore` from it. Filtered scans are unchanged (collected length, a lower bound). Layout-independent +(applies equally to branch/COW layouts). Regression: `tests/unit/storage/getNouns-totalCount.test.ts`. + +### Log — benign "mmap-vector backend not wired" downgraded to debug + +When a native vector provider replaces the JS HNSW index (it owns its own vector storage and exposes +no `setVectorBackend` hook), brainy logged `mmap-vector backend not wired … per-entity reads in use` +on **every** init. This is expected and benign in the native-index model — not a fault, and not by +itself an indication of per-entity reads — but it appeared on every warm and was repeatedly mistaken +for a cold-start cause. It is now a debug-level line (surface it with `BRAINY_LOG_LEVEL=debug`). + +> Note: this release fixes the misleading *count/log*. The remaining cold-start symptom on the +> native line (rebuilding instead of loading the persisted vector snapshot) is resolved when the +> native provider loads its snapshot at construction so the index reports a non-zero size before +> brainy's rebuild gate — already the model in the next major (8.0 + native 3.0). + +--- + ## v7.32.0 — 2026-06-16 **Affected products:** anyone who needs a **portable graph backup**, a partial export, or a diff --git a/src/brainy.ts b/src/brainy.ts index 33bcb91d..01ac6014 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -9473,12 +9473,17 @@ export class Brainy implements BrainyInterface { setVectorBackend?: (backend: MmapVectorBackend) => void } if (typeof indexWithBackend.setVectorBackend !== 'function') { - if (!this.config.silent) { - console.log( - '[brainy] mmap-vector backend not wired (vector index manages its own ' + - 'vector storage; no setVectorBackend hook) — per-entity reads in use' - ) - } + // Expected in the native-vector-index model: a native provider (e.g. + // @soulcraft/cortex) replaces the JS HNSW index and owns its own vector + // storage + persisted snapshot, so there is no setVectorBackend hook to + // wire here. This is benign, not a fault, and does NOT by itself imply + // per-entity reads — keep it at debug level so it never reads as a problem + // in normal operation. (The old console.log fired on every init and was + // repeatedly mistaken for the cold-start cause; see BRAINY-MMAP-VECTOR-HOOK.) + prodLog.debug( + '[brainy] mmap-vector backend not wired (native vector index manages ' + + 'its own vector storage; no setVectorBackend hook)' + ) return } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index c600038c..ec511292 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1515,11 +1515,28 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Apply pagination const paginatedNouns = collectedNouns.slice(offset, offset + limit) - const hasMore = collectedNouns.length > targetCount + + // totalCount must be the TRUE dataset total, not the size of this page. + // The shard scan above early-terminates at `targetCount = offset + limit` + // for memory efficiency, so `collectedNouns.length` only ever reaches the + // page size — returning it as `totalCount` made every non-empty brain look + // like it held exactly `limit` items. In particular + // `getNouns({ pagination: { limit: 1 } })` reported `totalCount: 1`, which + // tripped the index-rebuild gate into logging "Small dataset (1 items)" and + // rebuilding from scratch regardless of the real corpus size. For the + // unfiltered case the authoritative total is the O(1) counter maintained on + // every add/delete and rehydrated from `counts.json` on init; `Math.max` + // guards against a stale counter ever under-reporting below what we + // actually collected. A filtered scan has no cheap exact total, so it keeps + // the collected length (a lower bound — unchanged behaviour). + const totalCount = filter + ? collectedNouns.length + : Math.max(this.totalNounCount, collectedNouns.length) + const hasMore = offset + paginatedNouns.length < totalCount return { items: paginatedNouns, - totalCount: collectedNouns.length, + totalCount, hasMore, nextCursor: hasMore && paginatedNouns.length > 0 ? paginatedNouns[paginatedNouns.length - 1].id diff --git a/tests/unit/storage/getNouns-totalCount.test.ts b/tests/unit/storage/getNouns-totalCount.test.ts new file mode 100644 index 00000000..bddc6dbb --- /dev/null +++ b/tests/unit/storage/getNouns-totalCount.test.ts @@ -0,0 +1,88 @@ +/** + * @module tests/unit/storage/getNouns-totalCount + * @description Regression for BRAINY-MMAP-VECTOR-HOOK (Section H). The + * index-rebuild gate (`rebuildIndexesIfNeeded`) reads + * `getNouns({ pagination: { limit: 1 } }).totalCount` to decide whether a brain + * is "small" enough to rebuild inline. The shard-scan pagination + * (`BaseStorage.getNounsWithPagination`) early-terminates at `offset + limit` + * for memory efficiency, then returned `collectedNouns.length` as `totalCount` — + * i.e. the PAGE size, never the dataset total. So every non-empty filesystem + * brain reported `totalCount: 1` and logged "Small dataset (1 items)" on cold + * start, regardless of the real corpus size (a consumer saw this for an + * ~8.8k-entity brain). `totalCount` must be the authoritative O(1) noun counter + * (maintained on every add/delete, rehydrated from `counts.json` on init). + */ +import { describe, it, expect } from 'vitest' +import { mkdtempSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import type { NounMetadata } from '../../../src/coreTypes.js' + +const DIM = 8 + +/** A deterministic non-zero vector so the noun record is well-formed. */ +function vec(seed: number): number[] { + return Array.from({ length: DIM }, (_, i) => ((seed + i) % 7) / 7 - 0.5) +} + +/** Storage shards by UUID, so ids must be 32 hex chars. */ +function uuid(i: number): string { + return i.toString(16).padStart(32, '0') +} + +/** + * Seed `n` unique nouns. Metadata is saved BEFORE the noun record: it is the + * recommended order (avoids stat drift) and it is also where the O(1) noun + * counter is incremented, so this mirrors the real add() path. + */ +async function seed(storage: FileSystemStorage, n: number): Promise { + for (let i = 0; i < n; i++) { + const id = uuid(i) + await storage.saveNounMetadata(id, { + noun: 'thing', + createdAt: Date.now(), + updatedAt: Date.now() + } as NounMetadata) + await storage.saveNoun({ id, vector: vec(i), connections: new Map(), level: 0 }) + } +} + +describe('FileSystemStorage.getNouns totalCount (rebuild-gate count)', () => { + it('reports the TRUE total, not the page size, for a 1-item page', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-totalcount-')) + try { + const storage = new FileSystemStorage(dir) + await storage.init() + await seed(storage, 25) + + // The exact call the index-rebuild gate makes. + const onePage = await storage.getNouns({ pagination: { limit: 1 } }) + expect(onePage.items.length).toBe(1) + // Before the fix this was 1 (collectedNouns.length === limit). + expect(onePage.totalCount).toBe(25) + expect(onePage.hasMore).toBe(true) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('still reports the true total after a cold reopen (counts rehydrate)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-totalcount-reopen-')) + try { + const first = new FileSystemStorage(dir) + await first.init() + await seed(first, 25) + // Flush the count to counts.json so a fresh instance rehydrates it. + await (first as unknown as { persistCounts(): Promise }).persistCounts() + + // Cold start: a brand-new instance over the same directory. + const reopened = new FileSystemStorage(dir) + await reopened.init() + const page = await reopened.getNouns({ pagination: { limit: 1 } }) + expect(page.totalCount).toBe(25) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) From 5e7379dc4158494a7da7bacbe75464905806bb8f Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 17 Jun 2026 14:02:48 -0700 Subject: [PATCH 016/179] chore(release): 7.32.1 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdef7efd..d4a5e324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.32.1](https://github.com/soulcraftlabs/brainy/compare/v7.32.0...v7.32.1) (2026-06-17) + +- fix: getNouns().totalCount reports true total, not page size; quiet benign mmap-vector log (edff637) + + ### [7.32.0](https://github.com/soulcraftlabs/brainy/compare/v7.31.8...v7.32.0) (2026-06-16) - feat: portable graph export()/import() (BackupData v1) on brain.data() (a408d37) diff --git a/package-lock.json b/package-lock.json index 9909451c..0423ee98 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.32.0", + "version": "7.32.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.32.0", + "version": "7.32.1", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index 7ac5af92..f7245306 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.32.0", + "version": "7.32.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 89036deb20a7745f347a379094cf7726a98791a0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 19 Jun 2026 12:15:07 -0700 Subject: [PATCH 017/179] =?UTF-8?q?refactor:=20rename=20BackupData=20?= =?UTF-8?q?=E2=86=92=20PortableGraph=20(the=20type=20is=20interchange,=20n?= =?UTF-8?q?ot=20a=20backup)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parity with the 8.0 rename, backported to the 7.x line. The brain.data() export()/import() document type was BackupData, but it is the portable, versioned interchange representation of a graph (entities + relations + optional vectors), NOT a backup — exported for transport between instances, versions, and products. The actual backup is the native snapshot, so "Backup*" mis-signalled. Rename every developer-visible symbol, JSDoc, comment and doc: - BackupData→PortableGraph, BackupEntity→PortableGraphEntity, BackupRelation→PortableGraphRelation, BACKUP_FORMAT[_VERSION]→ PORTABLE_GRAPH_FORMAT[_VERSION], internal toBackup* helpers→toPortableGraph*. - src/api/DataAPI.ts, src/index.ts, src/cli/commands/core.ts, docs, and the test (renamed data-backup.test.ts → data-portable-graph.test.ts). The on-the-wire `format` tag is also renamed 'brainy-backup' → 'brainy-portable-graph': the export/import format was introduced in 7.32.0 and has not been adopted by any consumer, so there are no stored documents to stay compatible with — a clean rename beats carrying a legacy tag forward. No deprecated aliases (nothing to alias). 1505 unit green; build clean; data-portable-graph.test.ts 20/20. --- docs/api/README.md | 12 +- docs/architecture/storage-architecture.md | 10 +- docs/augmentations/EXAMPLES.md | 2 +- docs/guides/export-and-import.md | 30 ++--- docs/guides/framework-integration.md | 6 +- src/api/DataAPI.ts | 105 +++++++++--------- src/cli/commands/core.ts | 2 +- src/index.ts | 8 +- ...up.test.ts => data-portable-graph.test.ts} | 18 +-- 9 files changed, 98 insertions(+), 95 deletions(-) rename tests/unit/api/{data-backup.test.ts => data-portable-graph.test.ts} (96%) diff --git a/docs/api/README.md b/docs/api/README.md index 595a3bbd..e901742c 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -1765,18 +1765,18 @@ await brain.import('https://api.example.com/data.json') --- -### Export & Import (portable backup) +### Export & Import (portable graph) -`brain.data()` exposes a portable graph backup/restore API. `export(selector?, options?)` -serializes part or all of the graph to a versioned, portable `BackupData` document; -`import(backup, options?)` restores it (dedup-by-id merge by default, re-embedding when +`brain.data()` exposes a portable graph graph/restore API. `export(selector?, options?)` +serializes part or all of the graph to a versioned, portable `PortableGraph` document; +`import(graph, options?)` restores it (dedup-by-id merge by default, re-embedding when vectors are absent). ```typescript const data = await brain.data() -// Whole brain → a portable BackupData document -const backup = await data.export() +// Whole brain → a portable PortableGraph document +const graph = await data.export() // Just one workbench's members, with vectors, then restore elsewhere (merge by id) const subset = await data.export({ ids }, { includeVectors: true }) diff --git a/docs/architecture/storage-architecture.md b/docs/architecture/storage-architecture.md index fb9c0025..6fa945b5 100644 --- a/docs/architecture/storage-architecture.md +++ b/docs/architecture/storage-architecture.md @@ -457,14 +457,14 @@ await brain.storage.withLock('resource-id', async () => { ### Export Data ```typescript -// Export the whole brain to a portable BackupData document -const backup = await brain.data().then(d => d.export(undefined, { includeVectors: true })) +// Export the whole brain to a portable PortableGraph document +const graph = await brain.data().then(d => d.export(undefined, { includeVectors: true })) ``` ### Import Data ```typescript -// Restore a BackupData document (dedup-by-id merge by default) -await brain.data().then(d => d.import(backup, { onConflict: 'merge' })) +// Restore a PortableGraph document (dedup-by-id merge by default) +await brain.data().then(d => d.import(graph, { onConflict: 'merge' })) ``` See the [Export & Import guide](../guides/export-and-import.md) for partial exports @@ -479,7 +479,7 @@ const newBrain = new Brainy({ storage: { type: 's3' } }) await oldBrain.init() await newBrain.init() -// Transfer all data via a portable backup +// Transfer all data via a portable graph const data = await oldBrain.data().then(d => d.export(undefined, { includeVectors: true })) await newBrain.data().then(d => d.import(data)) ``` diff --git a/docs/augmentations/EXAMPLES.md b/docs/augmentations/EXAMPLES.md index 808139ba..24a98e64 100644 --- a/docs/augmentations/EXAMPLES.md +++ b/docs/augmentations/EXAMPLES.md @@ -546,7 +546,7 @@ const appComponent = await brain.findOne({ type: 'ReactComponent', name: 'App' } const imports = await brain.getRelated(appComponent.id, 'Imports') console.log(`App component imports:`, imports.map(c => c.name)) -// Export the analyzed component graph as a portable BackupData document +// Export the analyzed component graph as a portable PortableGraph document const graph = await brain.data().then(d => d.export({ type: 'ReactComponent' })) console.log(`Exported ${graph.stats.entityCount} components, ${graph.stats.relationCount} edges`) ``` diff --git a/docs/guides/export-and-import.md b/docs/guides/export-and-import.md index 3f367d33..c25b1fa8 100644 --- a/docs/guides/export-and-import.md +++ b/docs/guides/export-and-import.md @@ -5,7 +5,7 @@ public: true category: guides template: guide order: 9 -description: Export part or all of a brain to a portable, versioned BackupData document and import it back — by id, collection, connected neighbourhood, VFS subtree, predicate, or the whole brain. Covers vectors, edge policy, conflict handling, and id remapping. +description: Export part or all of a brain to a portable, versioned PortableGraph document and import it back — by id, collection, connected neighbourhood, VFS subtree, predicate, or the whole brain. Covers vectors, edge policy, conflict handling, and id remapping. next: - guides/subtypes-and-facets - api/README @@ -15,19 +15,19 @@ next: `brain.data()` exposes a **portable graph export/import** API. One method serializes a graph — an item, a collection, a connected neighbourhood, a VFS subtree, a predicate -match, or the whole brain — into a single versioned JSON document (`BackupData`); the +match, or the whole brain — into a single versioned JSON document (`PortableGraph`); the inverse restores it. ```typescript const data = await brain.data() -const backup = await data.export() // whole brain → BackupData -await data.import(backup) // restore (merge by id, re-embed if no vectors) +const graph = await data.export() // whole brain → PortableGraph +await data.import(graph) // restore (merge by id, re-embed if no vectors) ``` It is **portable** (human-readable JSON), **versioned** (`formatVersion`, so a 7.x export imports cleanly into 8.0), and **current-state** (the entities and edges as they are now — -no generation history). Use it for portable artifacts, partial backups, cross-environment +no generation history). Use it for portable artifacts, partial graphs, cross-environment moves, and version upgrades. ## When to use which @@ -38,13 +38,13 @@ moves, and version upgrades. | To ingest a CSV / PDF / Excel / JSON **file** as new entities | `brain.import(file)` — see [Import Anything](./import-anything.md) | The two are different operations that happen to share a verb: `import(file)` parses a -foreign document into new entities; `data().import(backup)` restores a graph this API +foreign document into new entities; `data().import(graph)` restores a graph this API exported. ## Exporting ```typescript -export(selector?, options?): Promise +export(selector?, options?): Promise ``` ### Selectors — *what* to export @@ -96,17 +96,17 @@ const tree = await data.export({ vfsPath: '/docs' }, { includeContent: true }) ## Importing ```typescript -import(backup, options?): Promise +import(graph, options?): Promise ``` ```typescript -const result = await brain.data().then(d => d.import(backup, { onConflict: 'merge' })) +const result = await brain.data().then(d => d.import(graph, { onConflict: 'merge' })) // → { imported, merged, skipped, reembedded, blobsWritten, errors } ``` | Option | Default | Effect | |--------|---------|--------| -| `onConflict` | `'merge'` | `'merge'` (update existing id in place — assemble many backups), `'replace'` (delete + recreate), or `'skip'`. | +| `onConflict` | `'merge'` | `'merge'` (update existing id in place — assemble many graphs), `'replace'` (delete + recreate), or `'skip'`. | | `reembed` | `'auto'` | `'auto'` (use the carried vector, else re-embed from `data`) or `'never'` (require a carried vector; record an error if absent). | | `remapIds` | — | Rewrite every id on the way in, e.g. to clone a template subgraph under fresh ids. | @@ -115,15 +115,15 @@ exported documents that share entity ids — re-importing an id merges rather th ```typescript // Clone a subgraph under fresh ids (a copy, not a move) -const remap = new Map(backup.entities.map(e => [e.id, crypto.randomUUID()])) -await data.import(backup, { remapIds: id => remap.get(id) ?? id }) +const remap = new Map(graph.entities.map(e => [e.id, crypto.randomUUID()])) +await data.import(graph, { remapIds: id => remap.get(id) ?? id }) ``` -## The `BackupData` format +## The `PortableGraph` format ```jsonc { - "format": "brainy-backup", + "format": "brainy-portable-graph", "formatVersion": 1, // import gates on this (cross-version migration) "brainyVersion": "7.32.0", "createdAt": "2026-06-16T…Z", @@ -154,7 +154,7 @@ of each entity; `metadata` holds **only** custom user fields — mirroring the i ## Cross-version (7.x → 8.0) -Because the document is shared and versioned, a backup written by 7.x imports into 8.0: +Because the document is shared and versioned, a graph written by 7.x imports into 8.0: `formatVersion` is read forward, `subtype` is carried so 8.0 re-types correctly, and the same 384-dimension model on both lines means `includeVectors:false` re-embeds identically (or `true` carries vectors verbatim). The format is **current-state** — if you need a diff --git a/docs/guides/framework-integration.md b/docs/guides/framework-integration.md index fab81011..7dd64bfe 100644 --- a/docs/guides/framework-integration.md +++ b/docs/guides/framework-integration.md @@ -540,11 +540,11 @@ export async function generateStaticProps() { }) await brain.init() - // Build a search index — export the whole brain as a portable BackupData document - const backup = await brain.data().then(d => d.export()) + // Build a search index — export the whole brain as a portable PortableGraph document + const graph = await brain.data().then(d => d.export()) return { - props: { searchIndex: backup.entities } + props: { searchIndex: graph.entities } } } ``` diff --git a/src/api/DataAPI.ts b/src/api/DataAPI.ts index b873a9ed..a88d0738 100644 --- a/src/api/DataAPI.ts +++ b/src/api/DataAPI.ts @@ -1,14 +1,17 @@ /** * @module DataAPI - * @description Portable graph backup/restore for Brainy — the `BackupData v1` - * export/import format plus `clear()`/`getStats()` data-management helpers. + * @description Portable graph export/import for Brainy — the `PortableGraph v1` + * format plus `clear()`/`getStats()` data-management helpers. * - * Accessed via `brain.data()`. The two headline methods are: + * A `PortableGraph` is the portable, versioned interchange representation of a graph + * (entities + relations + optional vectors), NOT a backup — exported for transport + * between instances, versions, and products. Accessed via `brain.data()`. The two + * headline methods are: * - * - **`export(selector?, options?) → BackupData`** — serialize a graph (an item, a + * - **`export(selector?, options?) → PortableGraph`** — serialize a graph (an item, a * collection + children, a connected neighbourhood, a VFS subtree, a predicate * match, or the whole brain) into ONE versioned, portable JSON document. - * - **`import(backup, options?) → ImportResult`** — restore a `BackupData` into the + * - **`import(graph, options?) → ImportResult`** — restore a `PortableGraph` into the * brain (dedup-by-id merge by default), re-embedding from `data` when vectors are * absent. * @@ -20,7 +23,7 @@ * * **Design decision (reserved-field split):** entities carry Brainy's standard fields * (`subtype`, `data`, `confidence`, `weight`, `service`, `createdBy`, `createdAt`) at - * the TOP LEVEL of each `BackupEntity`, and `metadata` holds ONLY custom user fields — + * the TOP LEVEL of each `PortableGraphEntity`, and `metadata` holds ONLY custom user fields — * mirroring the in-memory `Entity` shape, so `import()` maps each field to its dedicated * `add()`/`relate()` parameter rather than dumping everything into the metadata bag. */ @@ -32,10 +35,10 @@ import { getBrainyVersion } from '../utils/version.js' /** The fixed entity id of the VFS root collection (excluded from exports unless `includeSystem`). */ const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' -/** Magic string identifying a Brainy portable backup document. */ -const BACKUP_FORMAT = 'brainy-backup' +/** Magic string identifying a Brainy `PortableGraph` document (the `format` tag). */ +const PORTABLE_GRAPH_FORMAT = 'brainy-portable-graph' /** Current portable-format version. Import gates on this for cross-version migration. */ -const BACKUP_FORMAT_VERSION = 1 +const PORTABLE_GRAPH_FORMAT_VERSION = 1 /** Default embedding model label (informational; `dimensions` is the real compat gate). */ const DEFAULT_EMBED_MODEL = 'all-MiniLM-L6-v2' /** Storage-layer fetch ceiling for enumerations (well above any single-brain entity count). */ @@ -109,12 +112,12 @@ export interface ExportOptions { } /** - * @description Controls how a `BackupData` is applied to the brain on `import()`. + * @description Controls how a `PortableGraph` is applied to the brain on `import()`. */ export interface ImportOptions { /** * Conflict policy when an entity id already exists (default: `'merge'`): - * - `'merge'` — update in place (dedup-by-id; the default that lets you assemble many backups). + * - `'merge'` — update in place (dedup-by-id; the default that lets you assemble many graphs). * - `'replace'` — delete then re-create. * - `'skip'` — leave the existing entity untouched. */ @@ -129,8 +132,8 @@ export interface ImportOptions { remapIds?: (id: string) => string } -/** One entity in a `BackupData`. Standard fields top-level; `metadata` is custom-only. */ -export interface BackupEntity { +/** One entity in a `PortableGraph`. Standard fields top-level; `metadata` is custom-only. */ +export interface PortableGraphEntity { id: string /** NounType value. */ type: string @@ -156,8 +159,8 @@ export interface BackupEntity { metadata?: any } -/** One relation (edge) in a `BackupData`. */ -export interface BackupRelation { +/** One relation (edge) in a `PortableGraph`. */ +export interface PortableGraphRelation { id: string from: string to: string @@ -179,9 +182,9 @@ export interface BackupRelation { * @description A self-describing, versioned, portable graph document. The same shape * is produced/consumed on 7.x and 8.0; `formatVersion` gates cross-version migration. */ -export interface BackupData { - /** Always `'brainy-backup'` — identifies the document type. */ - format: typeof BACKUP_FORMAT +export interface PortableGraph { + /** Always `'brainy-portable-graph'` — identifies the document type. */ + format: typeof PORTABLE_GRAPH_FORMAT /** Integer format version (import gates on this). */ formatVersion: number /** The Brainy version that produced the document (informational). */ @@ -193,9 +196,9 @@ export interface BackupData { /** Echo of the selector that produced this document (provenance). */ selector?: ExportSelector /** Exported entities. */ - entities: BackupEntity[] + entities: PortableGraphEntity[] /** Exported relations. */ - relations: BackupRelation[] + relations: PortableGraphRelation[] /** VFS file bytes keyed by sha256 — present only with `includeContent`. */ blobs?: Record /** Endpoints referenced by `edges:'incident'` that fell outside the node set. */ @@ -244,10 +247,10 @@ export class DataAPI { // ============================================================================ /** - * @description Serialize part or all of the graph into a portable `BackupData`. + * @description Serialize part or all of the graph into a portable `PortableGraph`. * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}. * @param options - HOW to export (vectors, file bytes, edge policy). See {@link ExportOptions}. - * @returns A versioned, portable `BackupData` document. + * @returns A versioned, portable `PortableGraph` document. * @example * // A single workbench's members (exact id set), with vectors: * const backup = await brain.data().export({ ids }, { includeVectors: true }) @@ -261,7 +264,7 @@ export class DataAPI { * // The whole brain: * const backup = await brain.data().export() */ - async export(selector: ExportSelector = {}, options: ExportOptions = {}): Promise { + async export(selector: ExportSelector = {}, options: ExportOptions = {}): Promise { if (!this.brain) { throw new Error('DataAPI.export() requires a Brainy instance (use brain.data().export()).') } @@ -291,10 +294,10 @@ export class DataAPI { } // 4. Build entity records. - const entities: BackupEntity[] = [] + const entities: PortableGraphEntity[] = [] for (const id of idSet) { const e = entityMap.get(id) - if (e) entities.push(this.toBackupEntity(e, includeVectors)) + if (e) entities.push(this.toPortableGraphEntity(e, includeVectors)) } // 5. Collect edges per policy. @@ -315,8 +318,8 @@ export class DataAPI { const blobCount = blobs ? Object.keys(blobs).length : 0 return { - format: BACKUP_FORMAT, - formatVersion: BACKUP_FORMAT_VERSION, + format: PORTABLE_GRAPH_FORMAT, + formatVersion: PORTABLE_GRAPH_FORMAT_VERSION, brainyVersion: getBrainyVersion(), createdAt: new Date().toISOString(), embedding: { model: this.detectModel(), dimensions }, @@ -339,30 +342,30 @@ export class DataAPI { // ============================================================================ /** - * @description Restore a `BackupData` into the brain. Dedup-by-id merge by default, so + * @description Restore a `PortableGraph` into the brain. Dedup-by-id merge by default, so * assembling many backups that share entity ids merges rather than duplicates. Vectors * are re-embedded from `data` when absent (`reembed:'auto'`). - * @param data - A `BackupData` document (must have `format:'brainy-backup'`). + * @param data - A `PortableGraph` document (must have `format:'brainy-portable-graph'`). * @param options - Conflict/vector/id-remap policy. See {@link ImportOptions}. * @returns Counts of imported/merged/skipped/re-embedded entities + any per-record errors. - * @throws If `data` is not a `BackupData`, or its `formatVersion` is newer than supported. + * @throws If `data` is not a `PortableGraph`, or its `formatVersion` is newer than supported. * @example * const result = await brain.data().import(backup, { onConflict: 'merge' }) */ - async import(data: BackupData, options: ImportOptions = {}): Promise { + async import(data: PortableGraph, options: ImportOptions = {}): Promise { if (!this.brain) { throw new Error('DataAPI.import() requires a Brainy instance (use brain.data().import()).') } - if (!data || (data as any).format !== BACKUP_FORMAT) { + if (!data || (data as any).format !== PORTABLE_GRAPH_FORMAT) { throw new Error( - `DataAPI.import() expects a BackupData document (format:'${BACKUP_FORMAT}'). ` + + `DataAPI.import() expects a PortableGraph document (format:'${PORTABLE_GRAPH_FORMAT}'). ` + `For file ingestion (CSV/PDF/Excel/JSON), use brain.import() instead.` ) } - if (typeof data.formatVersion === 'number' && data.formatVersion > BACKUP_FORMAT_VERSION) { + if (typeof data.formatVersion === 'number' && data.formatVersion > PORTABLE_GRAPH_FORMAT_VERSION) { throw new Error( - `Backup formatVersion ${data.formatVersion} is newer than this Brainy supports ` + - `(max ${BACKUP_FORMAT_VERSION}). Upgrade Brainy to import this document.` + `PortableGraph formatVersion ${data.formatVersion} is newer than this Brainy supports ` + + `(max ${PORTABLE_GRAPH_FORMAT_VERSION}). Upgrade Brainy to import this document.` ) } @@ -698,9 +701,9 @@ export class DataAPI { // SERIALIZATION HELPERS (private) // ============================================================================ - /** Map a canonical `Entity` to a `BackupEntity` (reserved fields top-level). */ - private toBackupEntity(e: Entity, includeVectors: boolean): BackupEntity { - const be: BackupEntity = { id: e.id, type: e.type as string } + /** Map a canonical `Entity` to a `PortableGraphEntity` (reserved fields top-level). */ + private toPortableGraphEntity(e: Entity, includeVectors: boolean): PortableGraphEntity { + const be: PortableGraphEntity = { id: e.id, type: e.type as string } if (e.subtype !== undefined) be.subtype = e.subtype const vis = (e as any).visibility if (vis !== undefined && vis !== 'public') be.visibility = vis @@ -715,9 +718,9 @@ export class DataAPI { return be } - /** Map a canonical `Relation` to a `BackupRelation`. */ - private toBackupRelation(r: Relation): BackupRelation { - const br: BackupRelation = { id: r.id, from: r.from, to: r.to, type: r.type as string } + /** Map a canonical `Relation` to a `PortableGraphRelation`. */ + private toPortableGraphRelation(r: Relation): PortableGraphRelation { + const br: PortableGraphRelation = { id: r.id, from: r.from, to: r.to, type: r.type as string } if (r.subtype !== undefined) br.subtype = r.subtype const vis = (r as any).visibility if (vis !== undefined && vis !== 'public') br.visibility = vis @@ -731,10 +734,10 @@ export class DataAPI { private async collectEdges( idSet: Set, edges: 'induced' | 'incident' | 'none' - ): Promise<{ relations: BackupRelation[]; danglingIds?: string[] }> { + ): Promise<{ relations: PortableGraphRelation[]; danglingIds?: string[] }> { if (edges === 'none') return { relations: [] } - const relations: BackupRelation[] = [] + const relations: PortableGraphRelation[] = [] const dangling = new Set() const seen = new Set() @@ -747,7 +750,7 @@ export class DataAPI { if (edges === 'induced' && !toIn) continue if (!toIn) dangling.add(r.to) seen.add(r.id) - relations.push(this.toBackupRelation(r)) + relations.push(this.toPortableGraphRelation(r)) } } @@ -760,7 +763,7 @@ export class DataAPI { if (!idSet.has(r.from)) { dangling.add(r.from) seen.add(r.id) - relations.push(this.toBackupRelation(r)) + relations.push(this.toPortableGraphRelation(r)) } } } @@ -771,7 +774,7 @@ export class DataAPI { /** Read VFS file bytes (base64) for file entities, keyed by content hash. */ private async collectBlobs( - entities: BackupEntity[], + entities: PortableGraphEntity[], entityMap: Map ): Promise> { const blobs: Record = {} @@ -820,8 +823,8 @@ export class DataAPI { } } - /** `add()` params from a `BackupEntity` (excludes brain-managed fields like `createdAt`). */ - private entityAddFields(be: BackupEntity): Record { + /** `add()` params from a `PortableGraphEntity` (excludes brain-managed fields like `createdAt`). */ + private entityAddFields(be: PortableGraphEntity): Record { const fields: Record = { data: be.data, type: be.type as NounType @@ -834,8 +837,8 @@ export class DataAPI { return fields } - /** `update()` params from a `BackupEntity` (for `onConflict:'merge'`). */ - private entityUpdateFields(be: BackupEntity): Record { + /** `update()` params from a `PortableGraphEntity` (for `onConflict:'merge'`). */ + private entityUpdateFields(be: PortableGraphEntity): Record { const fields: Record = {} if (be.data !== undefined) fields.data = be.data if (be.type !== undefined) fields.type = be.type as NounType diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts index e0322bfe..76685bb7 100644 --- a/src/cli/commands/core.ts +++ b/src/cli/commands/core.ts @@ -876,7 +876,7 @@ export const coreCommands = { const brain = getBrainy() const format = options.format || 'json' - // Export the whole brain as a portable BackupData document (vectors + VFS file bytes). + // Export the whole brain as a portable PortableGraph document (vectors + VFS file bytes). const dataApi = await brain.data() const backup = await dataApi.export(undefined, { includeVectors: true, diff --git a/src/index.ts b/src/index.ts index 47e1b30b..8192fa15 100644 --- a/src/index.ts +++ b/src/index.ts @@ -116,12 +116,12 @@ export { // Export version utilities export { getBrainyVersion } from './utils/version.js' -// Export portable graph backup/restore API (brain.data()) +// Export portable graph export/import API (brain.data()) export { DataAPI } from './api/DataAPI.js' export type { - BackupData, - BackupEntity, - BackupRelation, + PortableGraph, + PortableGraphEntity, + PortableGraphRelation, ExportSelector, ExportOptions, ImportOptions, diff --git a/tests/unit/api/data-backup.test.ts b/tests/unit/api/data-portable-graph.test.ts similarity index 96% rename from tests/unit/api/data-backup.test.ts rename to tests/unit/api/data-portable-graph.test.ts index 2d3e88d1..7a1b4848 100644 --- a/tests/unit/api/data-backup.test.ts +++ b/tests/unit/api/data-portable-graph.test.ts @@ -1,7 +1,7 @@ /** * Unit tests for the portable graph backup/restore API (brain.data()). * - * Exercises BackupData v1 export()/import(): real round-trips against in-memory + * Exercises PortableGraph v1 export()/import(): real round-trips against in-memory * storage (no mocks of the API under test) — selectors, edge policy, vectors, * conflict handling, id remapping, subtype fidelity, and cross-version guards. */ @@ -11,11 +11,11 @@ import { randomUUID } from 'node:crypto' import { Brainy } from '../../../src/brainy' import { createTestConfig } from '../../helpers/test-factory' import { NounType, VerbType } from '../../../src/types/graphTypes' -import type { BackupData } from '../../../src/api/DataAPI' +import type { PortableGraph } from '../../../src/api/DataAPI' const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' -describe('DataAPI — portable graph backup/restore (BackupData v1)', () => { +describe('DataAPI — portable graph backup/restore (PortableGraph v1)', () => { let brain: Brainy beforeEach(async () => { @@ -28,14 +28,14 @@ describe('DataAPI — portable graph backup/restore (BackupData v1)', () => { }) describe('format + whole-brain round-trip', () => { - it('exports a self-describing, versioned BackupData document', async () => { + it('exports a self-describing, versioned PortableGraph document', async () => { const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' }) const b = await brain.add({ data: 'Acme', type: NounType.Organization }) await brain.relate({ from: a, to: b, type: VerbType.WorksWith, subtype: 'full-time' }) const backup = await brain.data().then((d) => d.export()) - expect(backup.format).toBe('brainy-backup') + expect(backup.format).toBe('brainy-portable-graph') expect(backup.formatVersion).toBe(1) expect(typeof backup.brainyVersion).toBe('string') expect(backup.brainyVersion.length).toBeGreaterThan(0) @@ -288,15 +288,15 @@ describe('DataAPI — portable graph backup/restore (BackupData v1)', () => { }) describe('import validation', () => { - it('rejects a non-BackupData payload', async () => { + it('rejects a non-PortableGraph payload', async () => { const d = await brain.data() - await expect(d.import({ entities: [] } as any)).rejects.toThrow(/BackupData/) + await expect(d.import({ entities: [] } as any)).rejects.toThrow(/PortableGraph/) }) it('rejects a newer formatVersion', async () => { const d = await brain.data() - const future: BackupData = { - format: 'brainy-backup', + const future: PortableGraph = { + format: 'brainy-portable-graph', formatVersion: 999, brainyVersion: 'x', createdAt: new Date().toISOString(), From c53dd61f96769861d70524da4916d0c52c36ec1f Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 19 Jun 2026 12:31:16 -0700 Subject: [PATCH 018/179] chore(release): 7.32.2 --- CHANGELOG.md | 5 +++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4a5e324..e37c8095 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [7.32.2](https://github.com/soulcraftlabs/brainy/compare/v7.32.1...v7.32.2) (2026-06-19) + +- refactor: rename BackupData → PortableGraph (the type is interchange, not a backup) (89036de) + + ### [7.32.1](https://github.com/soulcraftlabs/brainy/compare/v7.32.0...v7.32.1) (2026-06-17) - fix: getNouns().totalCount reports true total, not page size; quiet benign mmap-vector log (edff637) diff --git a/package-lock.json b/package-lock.json index 0423ee98..73f20d18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "7.32.1", + "version": "7.32.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "7.32.1", + "version": "7.32.2", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index f7245306..5fa80f6b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "7.32.1", + "version": "7.32.2", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", From 3a624454651761cf003e250269c8aece06dad0e3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Fri, 19 Jun 2026 16:40:35 -0700 Subject: [PATCH 019/179] feat: visibility tier (public/internal/system) on nouns + verbs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a reserved, top-level `visibility` field (mirrors the subtype rollout): 'public' (default, surfaced) | 'internal' (a consumer's app-internal data — hidden from default find()/getRelations()/counts/stats, opt-in via includeInternal) | 'system' (Brainy plumbing, library-set only; the add()/relate() param narrows to 'public' | 'internal'). Fixes a real leak: the VFS root entity counted in getNounCount() and appeared in find() (a fresh brain reported 1 entity). It is now visibility:'system' → excluded from every user-facing surface (fresh brain reports 0). - Reserved (STANDARD_ENTITY_FIELDS / STANDARD_VERB_FIELDS) — surfaced top-level on reads, never in the custom metadata bag; a 'public'/'internal' value smuggled through metadata is lifted to the field, 'system' dropped with a one-shot warning. - Threaded through add/relate/update/updateRelation + their internal mirrors; stored only when not 'public' so the common case stays lean. - Default exclusion in counts (baseStorage, isCountedVisibility), find()/getRelations() (hard candidate filter via excludeVisibility — keeps top-K/limit/offset correct), and rebuildCounts; includeInternal/includeSystem opt-ins. - VFS root marked 'system'. Same on-disk shape as the 8.0 line, so it carries forward unchanged. Backward-compatible: absent === 'public', so all existing data stays counted + returned. Tests: tests/unit/brainy/visibility.test.ts 17/17 (fresh-brain getNounCount()===0, internal hidden + opt-in, verb symmetry, top-level surfacing, metadata-spoof rejection). 1522 unit green; count-synchronization integration green. --- src/brainy.ts | 224 +++++++++++++++++++++++-- src/coreTypes.ts | 50 ++++++ src/storage/baseStorage.ts | 204 ++++++++++++++++++++--- src/types/brainy.types.ts | 86 +++++++++- src/utils/paramValidation.ts | 1 + src/utils/rebuildCounts.ts | 7 + src/vfs/VirtualFileSystem.ts | 9 ++ tests/unit/brainy/visibility.test.ts | 233 +++++++++++++++++++++++++++ 8 files changed, 778 insertions(+), 36 deletions(-) create mode 100644 tests/unit/brainy/visibility.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 01ac6014..3c1a6ac0 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -12,6 +12,7 @@ import { HNSWIndex } from './hnsw/hnswIndex.js' import { createStorage } from './storage/storageFactory.js' import { BaseStorage } from './storage/baseStorage.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' +import type { EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, cosineDistance, @@ -1038,6 +1039,14 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateAddParams(params) + // Reserved 'visibility' arriving via the metadata bag is normalized to the + // top-level param, mirroring add()'s lift behavior for the other reserved + // fields. A 'public'/'internal' value is user-settable and remaps to + // params.visibility (top-level wins when both are supplied); 'system' is + // Brainy-only and is dropped with a one-shot warning. In every case the key + // is stripped from metadata so it never echoes inside the custom bag. + params = this.remapReservedVisibilityOnAdd('add', params) + // Tracked-field vocabulary enforcement (Layer 2). Walks both bags so a // tracked field declared at top level (e.g. 'subtype') and one declared in // metadata (e.g. 'status') both validate. @@ -1084,6 +1093,9 @@ export class Brainy implements BrainyInterface { data: params.data, noun: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), service: params.service, createdAt: Date.now(), updatedAt: Date.now(), @@ -1105,6 +1117,8 @@ export class Brainy implements BrainyInterface { level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), createdAt: Date.now(), @@ -1378,6 +1392,7 @@ export class Brainy implements BrainyInterface { // Flatten common entity fields to top level type: entity.type, subtype: entity.subtype, + visibility: entity.visibility, metadata: entity.metadata, data: entity.data, confidence: entity.confidence, @@ -1408,6 +1423,7 @@ export class Brainy implements BrainyInterface { vector: noun.vector, type: noun.type || NounType.Thing, subtype: noun.subtype, + visibility: noun.visibility, // Standard fields at top-level confidence: noun.confidence, @@ -1451,13 +1467,14 @@ export class Brainy implements BrainyInterface { // Extract standard fields, rest are custom metadata // Same destructuring as baseStorage.getNoun() to ensure consistency - const { noun, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata + const { noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata const entity: Entity = { id, vector: [], // Stub vector (empty array - vectors not loaded for metadata-only) type: noun as NounType || NounType.Thing, subtype, + visibility: visibility as EntityVisibility | undefined, // Standard fields from metadata confidence, @@ -1544,12 +1561,13 @@ export class Brainy implements BrainyInterface { if (!params.metadata || typeof params.metadata !== 'object') return params const { - noun, subtype, createdAt, updatedAt, confidence, weight, service, + noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = params.metadata as Record const hasReserved = - noun !== undefined || subtype !== undefined || createdAt !== undefined || + noun !== undefined || subtype !== undefined || visibility !== undefined || + createdAt !== undefined || updatedAt !== undefined || confidence !== undefined || weight !== undefined || service !== undefined || data !== undefined || createdBy !== undefined || _rev !== undefined @@ -1572,16 +1590,125 @@ export class Brainy implements BrainyInterface { if (service !== undefined) warnDropped('service', 'nothing — fixed at add() time') if (createdBy !== undefined) warnDropped('createdBy', 'nothing — fixed at add() time') if (_rev !== undefined) warnDropped('_rev', "the 'ifRev' param for optimistic concurrency") + // 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal' + // value IS user-settable and is silently remapped to the top-level param below.) + if (visibility === 'system') { + warnDropped('visibility', "the 'visibility' param ('public' | 'internal')") + } return { ...params, metadata: customMetadata as UpdateParams['metadata'], ...(params.confidence === undefined && typeof confidence === 'number' && { confidence }), ...(params.weight === undefined && typeof weight === 'number' && { weight }), - ...(params.subtype === undefined && typeof subtype === 'string' && { subtype }) + ...(params.subtype === undefined && typeof subtype === 'string' && { subtype }), + ...(params.visibility === undefined && + (visibility === 'public' || visibility === 'internal') && { + visibility: visibility as 'public' | 'internal' + }) } } + /** + * @description Normalize a reserved `visibility` field that arrived inside the + * `metadata` bag of an `add()` / `relate()` call. Untyped (JavaScript) callers + * can smuggle the reserved key past the compile-time guard; this applies the + * same contract as the rest of the reserved fields: + * - `'public'` / `'internal'` — user-settable; remapped to the top-level + * `visibility` param unless the caller also passed it explicitly (top-level + * wins), and stripped from the metadata bag. + * - `'system'` — Brainy-only; dropped with a one-shot warning and stripped from + * the metadata bag (the entity/edge stays public). + * In every case the key is removed from `metadata` so it never echoes inside the + * custom bag. + * @param method - The calling method name (`'add'` | `'relate'`) for the warning text. + * @param params - The caller's params (not mutated; a normalized copy is returned). + * @returns Params with `visibility` normalized out of `metadata`. + */ + private remapReservedVisibilityOnAdd< + P extends { visibility?: 'public' | 'internal'; metadata?: any } + >(method: 'add' | 'relate', params: P): P { + if (!params.metadata || typeof params.metadata !== 'object') return params + const bag = params.metadata as Record + if (!('visibility' in bag)) return params + + const { visibility, ...customMetadata } = bag + if (visibility === 'system') { + this.warnDroppedReservedField( + method, + 'visibility', + "the 'visibility' param ('public' | 'internal')" + ) + } + + return { + ...params, + metadata: customMetadata, + ...(params.visibility === undefined && + (visibility === 'public' || visibility === 'internal') && { + visibility: visibility as 'public' | 'internal' + }) + } + } + + /** + * @description Emit a one-shot (per field, per process) warning that a reserved + * field supplied through a metadata bag was ignored, naming the correct write + * path. Shared by the add/relate/update reserved-field normalizers. + * @param method - The calling method name, used in the message. + * @param field - The reserved field that was dropped. + * @param rightPath - Human-readable description of where the value should go instead. + */ + private warnDroppedReservedField(method: string, field: string, rightPath: string): void { + if (Brainy.warnedReservedFields.has(field)) return + Brainy.warnedReservedFields.add(field) + prodLog.warn( + `[brainy] ${method}(): '${field}' is a reserved field and cannot be set ` + + `through a metadata patch — use ${rightPath}. The value was ignored. ` + + `(This warning is shown once per field per process.)` + ) + } + + /** + * The visibility tiers a read should EXCLUDE, given the caller's opt-ins. + * Default (no opts) hides both `'internal'` and `'system'`; `includeInternal` + * unhides internal; `includeSystem` unhides system. Returns `null` when nothing + * is excluded (both opts set) so callers can skip the filter entirely. + * + * @param params - The read params carrying `includeInternal` / `includeSystem`. + * @returns The set of excluded tier strings, or `null` for "exclude nothing". + */ + private excludedVisibilityTiers( + params: { includeInternal?: boolean; includeSystem?: boolean } + ): EntityVisibility[] | null { + const excluded: EntityVisibility[] = [] + if (!params.includeInternal) excluded.push('internal') + if (!params.includeSystem) excluded.push('system') + return excluded.length > 0 ? excluded : null + } + + /** + * Resolve the set of entity/relationship ids to hide for this read, by asking + * the metadata index for everything tagged with an excluded visibility tier. + * Public entities carry NO stored `visibility` (absent === public), so they are + * never in this set — exactly the entities we want to keep. Returns an empty set + * when nothing is excluded. The result is used as a HARD candidate filter + * (subtracted from candidate ids BEFORE pagination), so `limit`/`offset` stay correct. + * + * @param params - The read params carrying the visibility opt-ins. + * @returns A set of ids to omit from results. + */ + private async resolveHiddenIds( + params: { includeInternal?: boolean; includeSystem?: boolean } + ): Promise> { + const excluded = this.excludedVisibilityTiers(params) + if (!excluded) return new Set() + const ids = await this.metadataIndex.getIdsForFilter({ + visibility: excluded.length === 1 ? excluded[0] : { oneOf: excluded } + }) + return new Set(ids) + } + /** One-shot registry for reserved-field warnings (per process). */ private static warnedReservedFields = new Set() @@ -1672,7 +1799,13 @@ export class Brainy implements BrainyInterface { ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), // Update subtype if provided, otherwise preserve existing ...(params.subtype !== undefined && { subtype: params.subtype }), - ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }) + ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), + // Visibility: take the new value if provided, else preserve existing. Stored only + // when the effective value is not 'public' (absent === public, keeps records lean). + // A change to 'public' therefore drops the field entirely. + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }) } // Build entity structure for metadata index (with top-level fields) @@ -1683,6 +1816,9 @@ export class Brainy implements BrainyInterface { level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }), confidence: params.confidence !== undefined ? params.confidence : existing.confidence, weight: params.weight !== undefined ? params.weight : existing.weight, createdAt: existing.createdAt, @@ -1976,6 +2112,11 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateRelateParams(params) + // Reserved 'visibility' arriving via the metadata bag is normalized to the + // top-level param (mirror of add()): 'public'/'internal' lifts, 'system' is + // dropped with a one-shot warning, and the key is stripped from the bag. + params = this.remapReservedVisibilityOnAdd('relate', params) + // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered // via brain.requireSubtype() compose with the brain-wide strict-mode flag. // Metadata is passed so infrastructure edges (VFS containment) can bypass @@ -2028,6 +2169,9 @@ export class Brainy implements BrainyInterface { ...(params.metadata || {}), verb: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, createdAt: Date.now(), ...((params as any).data !== undefined && { data: (params as any).data }) @@ -2044,6 +2188,8 @@ export class Brainy implements BrainyInterface { verb: params.type, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, metadata: params.metadata, data: (params as any).data, @@ -2213,6 +2359,11 @@ export class Brainy implements BrainyInterface { ...(params.subtype !== undefined ? { subtype: params.subtype } : existingAny.subtype !== undefined && { subtype: existingAny.subtype }), + // Visibility: new value if provided, else preserve existing; stored only when the + // effective value is not 'public' (a change to 'public' drops the field). + ...(((params.visibility ?? existingAny.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existingAny.visibility + }), weight: params.weight ?? existingAny.weight ?? 1.0, ...(params.confidence !== undefined ? { confidence: params.confidence } @@ -2237,6 +2388,9 @@ export class Brainy implements BrainyInterface { ...(params.subtype !== undefined ? { subtype: params.subtype } : existingAny.subtype !== undefined && { subtype: existingAny.subtype }), + ...(((params.visibility ?? existingAny.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existingAny.visibility + }), weight: updatedMetadata.weight, metadata: newMetadata, data: updatedMetadata.data, @@ -2335,6 +2489,14 @@ export class Brainy implements BrainyInterface { filter.service = params.service } + // Visibility (8.0): exclude hidden tiers by default. The exclusion is applied in the + // storage scan AFTER metadata load (where verb.visibility is known), so the storage + // limit stays exact. Like subtype, it disqualifies the metadata-less fast paths. + const excludedTiers = this.excludedVisibilityTiers(params) + if (excludedTiers) { + filter.excludeVisibility = excludedTiers + } + // VFS relationships are no longer filtered // VFS is part of the knowledge graph - users can filter explicitly if needed @@ -3004,6 +3166,13 @@ export class Brainy implements BrainyInterface { return this.findAggregate(params) } + // Visibility (8.0): resolve the ids to hide for this read ONCE. Default hides + // internal + system; opt back in with includeInternal / includeSystem. Applied as a + // hard candidate filter at every candidate-gathering point below so limit/offset stay + // correct. Empty set when both opts are set (nothing excluded). + const hiddenIds = await this.resolveHiddenIds(params) + const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id) + const startTime = Date.now() const result = await (async () => { let results: Result[] = [] @@ -3079,6 +3248,9 @@ export class Brainy implements BrainyInterface { filteredIds = await this.metadataIndex.getIdsForFilter(filter) } + // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact. + if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) + // Paginate BEFORE loading entities (production-scale!) const limit = params.limit || 10 const offset = params.offset || 0 @@ -3105,7 +3277,8 @@ export class Brainy implements BrainyInterface { // Unfiltered sort: column store handles this at O(K log S) scale. // No per-entity storage reads, no bucketing precision loss. if (params.orderBy) { - const k = limit + offset + // Over-fetch by the hidden count so dropping hidden ids still fills the page. + const k = limit + offset + hiddenIds.size const sortedIntIds = await this.metadataIndex.columnStore.sortTopK( params.orderBy, params.order || 'asc', @@ -3114,9 +3287,11 @@ export class Brainy implements BrainyInterface { // Convert int IDs to UUIDs and paginate const idMapper = this.metadataIndex.getIdMapper() - const allUuids = sortedIntIds + let allUuids = sortedIntIds .map(intId => idMapper.getUuid(intId)) .filter((uuid): uuid is string => uuid !== undefined) + // Visibility hard filter — drop hidden ids BEFORE pagination. + if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) const entitiesMap = await this.batchGet(pageIds) @@ -3137,9 +3312,13 @@ export class Brainy implements BrainyInterface { filter.isVFSEntity = { ne: true } } - // Use metadata index if we need to filter + // Use metadata index when there's an actual filter (e.g. excludeVFS). The empty + // filter returns nothing from getIdsForFilter, so the unfiltered case below uses + // getNouns instead (it returns all nouns, including their visibility). if (Object.keys(filter).length > 0) { - const filteredIds = await this.metadataIndex.getIdsForFilter(filter) + let filteredIds = await this.metadataIndex.getIdsForFilter(filter) + // Visibility hard filter — drop hidden ids BEFORE pagination. + if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance @@ -3151,13 +3330,19 @@ export class Brainy implements BrainyInterface { } } } else { - // No filtering needed, use direct storage query + // No metadata filter, use direct storage query. Over-fetch by the hidden count + // so the visibility filter below still fills the requested page (limit-exact). const storageResults = await this.storage.getNouns({ - pagination: { limit: limit + offset, offset: 0 } + pagination: { limit: limit + offset + hiddenIds.size, offset: 0 } }) - for (let i = offset; i < Math.min(offset + limit, storageResults.items.length); i++) { - const noun = storageResults.items[i] + // Visibility hard filter on the materialized nouns (each carries `visibility`). + const visible = hiddenIds.size > 0 + ? storageResults.items.filter((noun) => noun && !hiddenIds.has(noun.id)) + : storageResults.items + + for (let i = offset; i < Math.min(offset + limit, visible.length); i++) { + const noun = visible[i] if (noun) { const entity = await this.convertNounToEntity(noun) results.push(this.createResult(noun.id, 1.0, entity)) @@ -3212,6 +3397,11 @@ export class Brainy implements BrainyInterface { } preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter) + // Visibility hard filter — restrict the HNSW candidate set to non-hidden ids. + if (hiddenIds.size > 0) { + preResolvedMetadataIds = preResolvedMetadataIds.filter((id) => !hiddenIds.has(id)) + } + // Short-circuit: if metadata filter matches nothing, skip expensive vector search if (preResolvedMetadataIds.length === 0) { return [] @@ -3275,6 +3465,13 @@ export class Brainy implements BrainyInterface { results = Array.from(uniqueResults.values()) } + // Visibility hard filter for vector/text/proximity candidates (the pre-resolved + // path already excluded hidden ids; this covers searches with no metadata filter). + // Applied BEFORE the final sort+slice so limit/offset remain exact. + if (hiddenIds.size > 0 && results.length > 0) { + results = results.filter((r) => !isHidden(r.id)) + } + // Apply metadata filtering using pre-resolved IDs (metadata-first optimization). // When vector search was performed, HNSW already filtered by candidateIds — // this block handles pagination, entity loading, and metadata-only queries. @@ -8821,6 +9018,7 @@ export class Brainy implements BrainyInterface { to: v.targetId, type: (v.verb || v.type) as VerbType, ...(va.subtype !== undefined && { subtype: va.subtype as string }), + ...(v.visibility !== undefined && { visibility: v.visibility }), weight: v.weight ?? 1.0, data: v.data, metadata: v.metadata, diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 5d55ec1f..765f58e4 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -204,6 +204,38 @@ export interface VerbMetadata { * * Used for API responses and storage retrieval. */ + +/** + * Visibility tier for an entity or relationship — controls whether it surfaces + * on Brainy's default user-facing read paths. A reserved, top-level field; the + * absence of the field is exactly equivalent to `'public'`. + * + * - `'public'` (DEFAULT, and the meaning when the field is absent) — counted and + * returned everywhere: `find()`, `related()`, `getNounCount()`/`getVerbCount()`, + * `stats()`. + * - `'internal'` — a consumer's app-internal data. HIDDEN from the default + * `find()` / `related()` / counts / `stats()`, but retrievable with an explicit + * opt-in (`find({ includeInternal: true })`, `related({ includeInternal: true })`). + * - `'system'` — Brainy's own plumbing (e.g. the VFS root entity). Hidden + * EVERYWHERE by default and surfaced only via the explicit `includeSystem` + * opt-in. NOT settable by consumer code — only internal Brainy code assigns it, + * which is why the `add()` / `relate()` params type narrows to + * `'public' | 'internal'`. + */ +export type EntityVisibility = 'public' | 'internal' | 'system' + +/** + * Whether an entity/relationship of the given visibility is counted on the + * user-facing surfaces (`getNounCount()`/`getVerbCount()`, per-type stats). Only + * `'public'` (and absent, which means public) is counted; `'internal'` and + * `'system'` are not. + * @param visibility - The stored `visibility` value (may be absent/unknown). + * @returns `true` when the entity is counted on the user-facing surfaces. + */ +export function isCountedVisibility(visibility: unknown): boolean { + return visibility !== 'internal' && visibility !== 'system' +} + export interface HNSWNounWithMetadata { // HNSW Core (unchanged) id: string @@ -222,6 +254,14 @@ export interface HNSWNounWithMetadata { // fast path, never falling through to the metadata fallback. subtype?: string + // VISIBILITY — three-value tier gating whether this entity surfaces on default + // user-facing reads. Absent === 'public' (counted + returned everywhere). 'internal' + // hides the entity from default find()/counts/stats but keeps it retrievable via + // find({ includeInternal: true }). 'system' (Brainy plumbing) is hidden everywhere + // unless find({ includeSystem: true }). Stored only when not 'public' so the common + // case stays lean. See {@link EntityVisibility}. + visibility?: EntityVisibility + // QUALITY METRICS (top-level, explicit) confidence?: number weight?: number @@ -264,6 +304,7 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet = new Set([ 'level', 'type', 'subtype', + 'visibility', 'confidence', 'weight', 'createdAt', @@ -323,6 +364,7 @@ export const STANDARD_VERB_FIELDS: ReadonlySet = new Set([ 'sourceId', 'targetId', 'subtype', + 'visibility', 'confidence', 'weight', 'createdAt', @@ -394,6 +436,13 @@ export interface HNSWVerbWithMetadata { // the metadata fallback. subtype?: string + // VISIBILITY — verb mirror of the entity tier. Absent === 'public' (counted + + // returned everywhere). 'internal' hides the edge from default related()/counts/stats + // but keeps it retrievable via related({ includeInternal: true }). 'system' is hidden + // everywhere unless related({ includeSystem: true }). Stored only when not 'public'. + // See {@link EntityVisibility}. + visibility?: EntityVisibility + // QUALITY METRICS (top-level, explicit) weight?: number confidence?: number @@ -427,6 +476,7 @@ export interface GraphVerb { connections?: Map> // Optional connections from HNSW index type?: string // Optional type of the relationship subtype?: string // Optional sub-classification within the VerbType (7.30+) + visibility?: EntityVisibility // Visibility tier; absent === 'public'. See {@link EntityVisibility}. weight?: number // Optional weight of the relationship confidence?: number // Optional confidence score (0-1) metadata?: any // Optional metadata for the verb diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index ec511292..c7daf1e8 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -13,8 +13,10 @@ import { VerbMetadata, HNSWNounWithMetadata, HNSWVerbWithMetadata, - StatisticsData + StatisticsData, + isCountedVisibility } from '../coreTypes.js' +import type { EntityVisibility } from '../coreTypes.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' import { validateNounType, validateVerbType } from '../utils/typeValidation.js' import { @@ -184,6 +186,18 @@ function getVerbMetadataPath(id: string): string { return `entities/verbs/${shard}/${id}/metadata.json` } +/** + * Extract the entity id from an ID-first metadata path. Both noun and verb + * metadata live at `entities////metadata.json`, so the id is + * the second-to-last path segment. + * @param path - A metadata.json path produced by `getNounMetadataPath` / `getVerbMetadataPath`. + * @returns The id segment, or `undefined` if the path doesn't have the expected shape. + */ +function idFromMetadataPath(path: string): string | undefined { + const segments = path.split('/') + return segments[segments.length - 2] +} + /** * Base storage adapter that implements common functionality * This is an abstract class that should be extended by specific storage adapters @@ -278,6 +292,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected verbSubtypeByIdCache = new Map() // Total: 676 bytes (99.2% reduction vs Map-based tracking) + /** + * In-memory map from noun id → its non-public `visibility` tier ('internal' | + * 'system'). Parallel to `nounTypeByIdCache`. Lets `saveNoun_internal()` + * (which has no metadata access in its signature) skip the user-facing + * `nounCountsByType` increment for hidden entities, and lets + * `deleteNounMetadata()` skip the matching decrement — keeping the counts + * symmetric. Sparse by design: public entities (the common case) get NO + * entry, so the absence of an id means "public, counted". + */ + protected nounVisibilityByIdCache = new Map() + + /** + * In-memory map from verb id → its non-public `visibility` tier. Verb-side + * mirror of `nounVisibilityByIdCache`. Sparse — public edges get no entry. + */ + protected verbVisibilityByIdCache = new Map() + // Type caches REMOVED - ID-first paths eliminate need for type lookups! // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json // Type is just a field in the metadata, indexed by MetadataIndexManager for queries @@ -904,7 +935,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } // Combine into HNSWNounWithMetadata - Extract standard fields to top-level - const { noun, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata + const { noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata return { id: vector.id, @@ -914,6 +945,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Standard fields at top-level type: (noun as NounType) || NounType.Thing, subtype: subtype as string | undefined, + visibility: visibility as EntityVisibility | undefined, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -943,13 +975,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { for (const noun of nouns) { const metadata = await this.getNounMetadata(noun.id) if (metadata) { - const { noun: nounType, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata + const { noun: nounType, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata nounsWithMetadata.push({ ...noun, // Standard fields at top-level type: (nounType as NounType) || NounType.Thing, subtype: subtype as string | undefined, + visibility: visibility as EntityVisibility | undefined, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -1023,7 +1056,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } // Combine into HNSWVerbWithMetadata - Extract standard fields to top-level - const { subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata + const { subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata return { id: verb.id, @@ -1034,6 +1067,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId: verb.targetId, // Standard fields at top-level subtype: subtype as string | undefined, + visibility: visibility as EntityVisibility | undefined, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -1098,7 +1132,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const verb = this.deserializeVerb(vectorData) // Extract standard fields to top-level - const { subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadataData + const { subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadataData results.set(id, { id: verb.id, @@ -1109,6 +1143,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId: verb.targetId, // Standard fields at top-level subtype: subtype as string | undefined, + visibility: visibility as EntityVisibility | undefined, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -1489,6 +1524,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { ...deserialized, type: (metadata.noun || 'thing') as NounType, subtype: (metadata as any).subtype as string | undefined, + visibility: (metadata as any).visibility as EntityVisibility | undefined, confidence: metadata.confidence, weight: metadata.weight, createdAt: metadata.createdAt @@ -1569,6 +1605,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId?: string | string[] service?: string | string[] metadata?: Record + /** + * 8.0 visibility: tiers to exclude from results (e.g. `['internal','system']`). + * Applied after metadata load in the full scan, so it disqualifies the + * metadata-less graph-index fast paths (same as `subtype`). + */ + excludeVisibility?: string[] } }): Promise<{ items: HNSWVerbWithMetadata[] @@ -1599,6 +1641,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { : [(filter as any).subtype] ) : null + // 8.0 visibility exclusion — applied after metadata load (verb visibility lives in + // metadata), so hidden edges are skipped BEFORE the pagination window fills. + const excludeVisibility = (filter as { excludeVisibility?: string[] } | undefined)?.excludeVisibility + const filterExcludeVisibility = excludeVisibility && excludeVisibility.length > 0 + ? new Set(excludeVisibility) + : null // Iterate by shards (0x00-0xFF) instead of types - single pass! for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) { @@ -1645,10 +1693,20 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } + // Apply visibility exclusion (8.0). Absent === 'public' (kept); a stored + // 'internal'/'system' value is dropped when its tier is excluded. + if (filterExcludeVisibility) { + const visibility = (metadata as any)?.visibility as string | undefined + if (visibility && filterExcludeVisibility.has(visibility)) { + continue + } + } + // Combine verb + metadata collectedVerbs.push({ ...verb, subtype: (metadata as any)?.subtype as string | undefined, + visibility: (metadata as any)?.visibility as EntityVisibility | undefined, weight: metadata?.weight, confidence: metadata?.confidence, createdAt: metadata?.createdAt @@ -1701,6 +1759,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId?: string | string[] service?: string | string[] metadata?: Record + /** + * 8.0 visibility: tiers to exclude from results (e.g. `['internal','system']`). + * Applied after metadata load in the full scan, so it disqualifies the + * metadata-less graph-index fast paths (same as `subtype`). + */ + excludeVisibility?: string[] } }): Promise<{ items: HNSWVerbWithMetadata[] @@ -1721,8 +1785,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { // fallthrough to the full shard scan (which loads metadata and applies the // subtype check after the load). Subtype is not stored on the raw HNSWVerb; // it's a metadata field. The graph-index fast paths return verbs without - // metadata, so they can't apply subtype filtering correctly. - if (options?.filter && !(options.filter as any).subtype) { + // metadata, so they can't apply subtype filtering correctly. The 8.0 + // `excludeVisibility` filter (also a metadata field) disqualifies them the same way. + if ( + options?.filter && + !(options.filter as any).subtype && + !(options.filter as { excludeVisibility?: string[] }).excludeVisibility + ) { // CRITICAL VFS FIX: If filtering by sourceId + verbType (most common VFS pattern!) // This is the query PathResolver.getChildren() uses: getRelations({ from: dirId, type: VerbType.Contains }) if ( @@ -2311,16 +2380,40 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.nounSubtypeByIdCache.delete(id) } + // Visibility (8.0): only public entities count toward the user-facing totals. + // Warm `nounVisibilityByIdCache` so `saveNoun_internal()` (no metadata access) + // can gate the `nounCountsByType` increment, and `deleteNounMetadata()` can gate + // the matching decrement. Sparse — public ids get no entry. + const newVisibility = metadata.visibility + const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) + const isCounted = isCountedVisibility(newVisibility) + if (isCounted) { + this.nounVisibilityByIdCache.delete(id) + } else { + this.nounVisibilityByIdCache.set(id, newVisibility as EntityVisibility) + } + // CRITICAL FIX: Increment count for new entities // This runs AFTER metadata is saved, guaranteeing type information is available // Uses synchronous increment since storage operations are already serialized // Fixes Bug #1: Count synchronization failure during add() and import() - if (isNew && metadata.noun) { + // 8.0: skip the user-facing total for internal/system entities (counts.json + getNounCount()). + if (isNew && metadata.noun && isCounted) { this.incrementEntityCount(metadata.noun) // Persist counts asynchronously (fire and forget) this.scheduleCountPersist().catch(() => { // Ignore persist errors - will retry on next operation }) + } else if (!isNew && metadata.noun && wasCounted !== isCounted) { + // Visibility flipped on update(): move the entity in/out of the user-facing total + // (counts.json / getNounCount()). `nounCountsByType` is gated independently in + // `saveNoun_internal()` off the cache warmed just above, so it is not touched here. + if (isCounted) { + this.incrementEntityCount(metadata.noun) + } else { + this.decrementEntityCount(metadata.noun) + } + this.scheduleCountPersist().catch(() => {}) } } @@ -2707,11 +2800,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { // decrement `nounCountsByType` so type-statistics stay honest across // deletes; symmetric with the increment in `saveNounMetadata_internal()`. const priorType = this.nounTypeByIdCache.get(id) + // 8.0 visibility: an internal/system entity was never added to `nounCountsByType` + // (gated in `saveNoun_internal()`), so it must not be decremented here either. + const priorCounted = isCountedVisibility(this.nounVisibilityByIdCache.get(id)) + this.nounVisibilityByIdCache.delete(id) if (priorType) { this.nounTypeByIdCache.delete(id) - const idx = TypeUtils.getNounIndex(priorType) - if (this.nounCountsByType[idx] > 0) { - this.nounCountsByType[idx]-- + if (priorCounted) { + const idx = TypeUtils.getNounIndex(priorType) + if (this.nounCountsByType[idx] > 0) { + this.nounCountsByType[idx]-- + } } // Symmetric subtype decrement @@ -2797,16 +2896,52 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.verbSubtypeByIdCache.delete(id) } + // Visibility (8.0): verb mirror of the noun count gating. Warm + // `verbVisibilityByIdCache` so `deleteVerbMetadata()` can prune it. Sparse — + // public edges get no entry. + // + // NOTE on `verbCountsByType`: unlike the noun path, `saveVerb_internal()` runs + // BEFORE this method (relate() saves the verb vector first) and has already done + // an UNCONDITIONAL `verbCountsByType[idx]++`. We therefore COMPENSATE here: for a + // new hidden edge, undo that bump. `updateRelation()` does not re-run + // `saveVerb_internal()`, so on a visibility flip we adjust the bucket directly. + const newVisibility = metadata.visibility + const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) + const isCounted = isCountedVisibility(newVisibility) + if (isCounted) { + this.verbVisibilityByIdCache.delete(id) + } else { + this.verbVisibilityByIdCache.set(id, newVisibility as EntityVisibility) + } + const verbTypeIdx = TypeUtils.getVerbIndex(verbType) + // CRITICAL FIX: Increment verb count for new relationships // This runs AFTER metadata is saved // Uses synchronous increment since storage operations are already serialized // Fixes Bug #2: Count synchronization failure during relate() and import() + // 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()). if (isNew) { - this.incrementVerbCount(verbType) + if (isCounted) { + this.incrementVerbCount(verbType) + } else { + // Hidden edge: undo the unconditional bump from saveVerb_internal(). + if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]-- + } // Persist counts asynchronously (fire and forget) this.scheduleCountPersist().catch(() => { // Ignore persist errors - will retry on next operation }) + } else if (wasCounted !== isCounted) { + // Visibility flipped on updateRelation() (saveVerb_internal did not run): move the + // edge in/out of both the user-facing total and the per-type bucket. + if (isCounted) { + this.incrementVerbCount(verbType) + this.verbCountsByType[verbTypeIdx]++ + } else { + this.decrementVerbCount(verbType) + if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]-- + } + this.scheduleCountPersist().catch(() => {}) } } @@ -2845,6 +2980,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.verbSubtypeByIdCache.delete(id) this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype) } + // 8.0 visibility: prune the cache entry (verb deletes don't touch verbCountsByType + // in this path, mirroring the existing verb-subtype delete semantics). + this.verbVisibilityByIdCache.delete(id) } // ============================================================================ @@ -3276,9 +3414,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { const metadata = await this.readWithInheritance(path) if (metadata && metadata.noun) { - const typeIndex = TypeUtils.getNounIndex(metadata.noun) - if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { - this.nounCountsByType[typeIndex]++ + // 8.0 visibility: rebuild only public entities into the user-facing + // per-type stat; warm the cache so later writes/deletes stay symmetric. + const id = idFromMetadataPath(path) + if (isCountedVisibility(metadata.visibility)) { + const typeIndex = TypeUtils.getNounIndex(metadata.noun) + if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { + this.nounCountsByType[typeIndex]++ + } + if (id) this.nounVisibilityByIdCache.delete(id) + } else if (id) { + this.nounVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) } } } catch (error) { @@ -3304,9 +3450,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { const metadata = await this.readWithInheritance(path) if (metadata && metadata.verb) { - const typeIndex = TypeUtils.getVerbIndex(metadata.verb) - if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { - this.verbCountsByType[typeIndex]++ + // 8.0 visibility: rebuild only public edges into the user-facing per-type stat. + const id = idFromMetadataPath(path) + if (isCountedVisibility(metadata.visibility)) { + const typeIndex = TypeUtils.getVerbIndex(metadata.verb) + if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { + this.verbCountsByType[typeIndex]++ + } + if (id) this.verbVisibilityByIdCache.delete(id) + } else if (id) { + this.verbVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) } } } catch (error) { @@ -3483,17 +3636,24 @@ export abstract class BaseStorage extends BaseStorageAdapter { const type = this.getNounType(noun) const path = getNounVectorPath(noun.id) - // Update type tracking + // Update type tracking — but only for entities that count toward the + // user-facing per-type stats. `nounVisibilityByIdCache` (warmed by + // `saveNounMetadata_internal()`) flags internal/system ids; public ids are + // absent, so the common case still increments. 8.0 visibility exclusion. const typeIndex = TypeUtils.getNounIndex(type) - this.nounCountsByType[typeIndex]++ + const counted = isCountedVisibility(this.nounVisibilityByIdCache.get(noun.id)) + if (counted) { + this.nounCountsByType[typeIndex]++ + } // COW-aware write: Use COW helper for branch isolation await this.writeObjectToBranch(path, noun) // Periodically save statistics // Also save on first noun of each type to ensure low-count types are tracked - const shouldSave = this.nounCountsByType[typeIndex] === 1 || // First noun of type - this.nounCountsByType[typeIndex] % 100 === 0 // Every 100th + const shouldSave = counted && + (this.nounCountsByType[typeIndex] === 1 || // First noun of type + this.nounCountsByType[typeIndex] % 100 === 0) // Every 100th if (shouldSave) { await this.saveTypeStatistics() } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 70055caa..9da3b32b 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -5,6 +5,7 @@ */ import { Vector } from '../coreTypes.js' +import type { EntityVisibility } from '../coreTypes.js' import { NounType, VerbType } from './graphTypes.js' // ============= Core Types ============= @@ -33,6 +34,13 @@ export interface Entity { * rolled into per-NounType statistics. */ subtype?: string + /** + * Visibility tier (reserved top-level field). Absent === `'public'` (counted and + * returned everywhere). `'internal'` hides the entity from default `find()` / counts / + * `stats()` (opt back in with `find({ includeInternal: true })`); `'system'` is Brainy + * plumbing, hidden unless `find({ includeSystem: true })`. See {@link EntityVisibility}. + */ + visibility?: EntityVisibility /** Opaque content — used for embeddings and semantic search. Not indexed by MetadataIndex. */ data?: any /** User-defined structured fields — indexed and queryable via `where` filters. */ @@ -84,6 +92,14 @@ export interface Relation { * the column-store directly. */ subtype?: string + /** + * Visibility tier (reserved top-level field), the verb mirror of `Entity.visibility`. + * Absent === `'public'` (counted and returned everywhere). `'internal'` hides the edge + * from default `related()` / counts / `stats()` (opt back in with + * `related({ includeInternal: true })`); `'system'` is Brainy plumbing. See + * {@link EntityVisibility}. + */ + visibility?: EntityVisibility /** Connection strength (0-1, default: 1.0) */ weight?: number /** Opaque content for the relationship (overrides auto-computed vector if provided) */ @@ -129,6 +145,7 @@ export interface Result { // Convenience: Common entity fields flattened to top level type?: NounType // Entity type (from entity.type) subtype?: string // Per-product sub-classification (from entity.subtype) + visibility?: EntityVisibility // Visibility tier (from entity.visibility); absent === 'public' metadata?: T // Entity metadata (from entity.metadata) data?: any // Entity data (from entity.data) confidence?: number // Type classification confidence (from entity.confidence) @@ -191,6 +208,15 @@ export interface AddParams { * aggregation on the standard-field fast path. */ subtype?: string + /** + * Visibility tier (reserved top-level field). Omit (or pass `'public'`) for normal + * data that is counted and returned everywhere. Pass `'internal'` for app-internal + * data that should be hidden from default `find()` / counts / `stats()` but stay + * retrievable via `find({ includeInternal: true })`. The `'system'` tier is reserved + * for Brainy's own plumbing and is intentionally not accepted here. Stored only when + * not `'public'`. + */ + visibility?: 'public' | 'internal' /** Structured queryable fields — indexed by MetadataIndex, used in `where` filters */ metadata?: T /** Custom entity ID (auto-generated UUID v4 if not provided) */ @@ -222,6 +248,13 @@ export interface UpdateParams { data?: any // New content to re-embed type?: NounType // Change type subtype?: string // Change subtype (set to '' or null-equivalent via dedicated unset is future work) + /** + * Change the visibility tier. Omit to preserve the existing value. Toggling between + * `'public'` and `'internal'` moves the entity in/out of the default-visible counts and + * `find()` results. The `'system'` tier is Brainy-internal (e.g. re-asserted on the VFS + * root during repair) — consumer code should only ever use `'public'` / `'internal'`. + */ + visibility?: EntityVisibility metadata?: Partial // Metadata to update merge?: boolean // Merge or replace metadata (default: true) vector?: Vector // New pre-computed vector @@ -259,6 +292,15 @@ export interface RelateParams { * (`getRelations({ verb, subtype })`) and aggregation (`groupBy:['subtype']`). */ subtype?: string + /** + * Visibility tier (reserved top-level field), the verb mirror of `AddParams.visibility`. + * Omit (or pass `'public'`) for normal edges that are counted and returned everywhere. + * Pass `'internal'` for app-internal edges hidden from default `related()` / counts / + * `stats()` but retrievable via `related({ includeInternal: true })`. The `'system'` + * tier is reserved for Brainy's own plumbing and is not accepted here. Stored only when + * not `'public'`. + */ + visibility?: 'public' | 'internal' /** Connection strength (0-1, default: 1.0) */ weight?: number /** Content for the relationship (optional — overrides auto-computed vector) */ @@ -282,6 +324,12 @@ export interface UpdateRelationParams { id: string // Relation to update type?: VerbType // Change verb type subtype?: string // Change sub-classification (omit to preserve existing) + /** + * Change the visibility tier. Omit to preserve the existing value. The verb mirror of + * `UpdateParams.visibility`; `'system'` is Brainy-internal — consumer code should only + * use `'public'` / `'internal'`. + */ + visibility?: EntityVisibility weight?: number // New weight confidence?: number // New confidence (0-1) data?: any // New content @@ -320,7 +368,24 @@ export interface FindParams { subtype?: string | string[] /** Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`) */ where?: Partial - + + // Visibility + /** + * Also return `visibility: 'internal'` entities. By default `find()` returns ONLY + * public entities (those with no `visibility` field, or `visibility: 'public'`); + * app-internal entities are hidden. Set `true` to include them. Applied as a hard + * candidate filter, so `limit` / `offset` stay correct. Does NOT include `'system'` + * entities — use `includeSystem` for those. + */ + includeInternal?: boolean + /** + * Also return `visibility: 'system'` entities (Brainy's own plumbing, e.g. the VFS + * root). Hidden by default and even when `includeInternal` is set. Set `true` only + * when you specifically need to see system entities. Applied as a hard candidate + * filter, so `limit` / `offset` stay correct. + */ + includeSystem?: boolean + // Graph Intelligence connected?: GraphConstraints @@ -481,6 +546,25 @@ export interface GetRelationsParams { */ subtype?: string | string[] + /** + * Also return `visibility: 'internal'` relationships. + * + * By default `related()` returns ONLY public relationships (those with no + * `visibility` field, or `visibility: 'public'`); app-internal edges are hidden. + * Set `true` to include them. Applied as a hard candidate filter so `limit` / + * `offset` stay correct. Does NOT include `'system'` edges — use `includeSystem`. + */ + includeInternal?: boolean + + /** + * Also return `visibility: 'system'` relationships (Brainy's own plumbing). + * + * Hidden by default and even when `includeInternal` is set. Set `true` only when + * you specifically need system edges. Applied as a hard candidate filter so + * `limit` / `offset` stay correct. + */ + includeSystem?: boolean + /** * Maximum number of results to return * diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 86b06ea5..771f2293 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -534,6 +534,7 @@ export function validateUpdateParams(params: UpdateParams): void { !params.type && !params.vector && params.subtype === undefined && + params.visibility === undefined && params.confidence === undefined && params.weight === undefined ) { diff --git a/src/utils/rebuildCounts.ts b/src/utils/rebuildCounts.ts index 389e7237..ce821772 100644 --- a/src/utils/rebuildCounts.ts +++ b/src/utils/rebuildCounts.ts @@ -75,6 +75,11 @@ export async function rebuildCounts(storage: BaseStorage): Promise