diff --git a/docs/concepts/consistency-model.md b/docs/concepts/consistency-model.md index 709978f9..db8efd91 100644 --- a/docs/concepts/consistency-model.md +++ b/docs/concepts/consistency-model.md @@ -261,6 +261,7 @@ never appear inside a `metadata` bag: |---|---|---| | `noun` | `verb` | the `type` param of `add()` / `relate()` | | `subtype` | `subtype` | the `subtype` param | +| `visibility` | `visibility` | the `visibility` param (`'public'` \| `'internal'`) | | `confidence` | `confidence` | the `confidence` param | | `weight` | `weight` | the `weight` param | | `service` | `service` | the `service` param (fixed at create time) | @@ -300,6 +301,54 @@ entity.confidence // 0.95 — top level entity.metadata // { customer: 'acme', total: 129.5 } ``` +### Visibility — `public` / `internal` / `system` + +`visibility` is a reserved tier that controls whether an entity or relationship +surfaces on Brainy's **default** user-facing reads. The absence of the field is +exactly equivalent to `'public'`. + +| Tier | Counted in `getNounCount()` / `stats()`? | Returned by default `find()` / `related()`? | Opt-in | +|---|---|---|---| +| `'public'` (default, or field absent) | yes | yes | — | +| `'internal'` | no | no | `find({ includeInternal: true })` / `related({ includeInternal: true })` | +| `'system'` | no | no | `find({ includeSystem: true })` / `related({ includeSystem: true })` | + +- **`'public'`** — normal data. Counted and returned everywhere. Stored lean: + the field is omitted on disk for public records, so existing data needs no + migration. +- **`'internal'`** — your app's own bookkeeping (audit trails, derived caches, + scratch entities) that should not pollute default queries, counts, or + `stats()`, yet must stay retrievable on demand. Set it via the `visibility` + param; read it back with the `includeInternal` opt-in. +- **`'system'`** — Brainy's own plumbing (for example the Virtual File System + root entity). Hidden everywhere by default — even when `includeInternal` is + set — and surfaced only with the explicit `includeSystem` opt-in. The + `'system'` tier is **not** part of the public `add()` / `relate()` param type + (`'public' | 'internal'`); only internal Brainy code assigns it. + +The opt-ins are applied as a **hard candidate filter** — hidden entities are +removed before `limit` / `offset` are applied, so a default `find({ limit: 10 })` +always returns ten *visible* results when that many exist, never a short page. + +```typescript +// App-internal scratch entity: present, retrievable, but out of the way. +await brain.add({ type: 'task', data: 'reindex job', visibility: 'internal' }) + +await brain.getNounCount() // unchanged — internal not counted +await brain.find({ type: 'task' }) // [] — hidden by default +await brain.find({ type: 'task', includeInternal: true }) // includes it + +// A brand-new brain reports zero user entities even though the VFS root exists: +const fresh = new Brainy() +await fresh.init() +await fresh.getNounCount() // 0 — the root is visibility:'system' +``` + +> **Note (8.0):** the structural `Contains` edges the VFS creates between +> directories and files are left at the default (`public`) visibility for now — +> only the VFS *root entity* is `'system'`. Marking those edges system requires +> companion changes to VFS traversal and is out of scope for this change. + ## What is not guaranteed Stated plainly, so nothing surprises you in production: diff --git a/src/brainy.ts b/src/brainy.ts index a5ab13cc..63398c99 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -14,7 +14,7 @@ import type { StorageOptions } from './storage/storageFactory.js' import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js' import { BaseStorage } from './storage/baseStorage.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' -import type { HNSWNounWithMetadata, HNSWVerbWithMetadata } from './coreTypes.js' +import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, cosineDistance, @@ -1248,6 +1248,9 @@ export class Brainy implements BrainyInterface { data: params.data, noun: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), service: params.service, createdAt: Date.now(), updatedAt: Date.now(), @@ -1269,6 +1272,8 @@ export class Brainy implements BrainyInterface { level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), createdAt: Date.now(), @@ -1542,6 +1547,7 @@ export class Brainy implements BrainyInterface { // Flatten common entity fields to top level type: entity.type, subtype: entity.subtype, + visibility: entity.visibility, metadata: entity.metadata, data: entity.data, confidence: entity.confidence, @@ -1572,6 +1578,7 @@ export class Brainy implements BrainyInterface { vector: noun.vector, type: noun.type || NounType.Thing, subtype: noun.subtype, + visibility: noun.visibility, // Standard fields at top-level confidence: noun.confidence, @@ -1622,6 +1629,7 @@ export class Brainy implements BrainyInterface { vector: [], // Stub vector (empty array - vectors not loaded for metadata-only) type: (reserved.noun as NounType) || NounType.Thing, subtype: reserved.subtype as string | undefined, + visibility: reserved.visibility as EntityVisibility | undefined, // Standard fields from metadata confidence: reserved.confidence as number | undefined, @@ -1703,6 +1711,11 @@ export class Brainy implements BrainyInterface { : 'nothing — revisions are system-managed' ) } + // 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal' + // value IS user-settable and is silently remapped to the top-level param below.) + if (reserved.visibility === 'system') { + this.warnDroppedReservedField(method, 'visibility', "the 'visibility' param ('public' | 'internal')") + } if (method === 'update') { if (reserved.service !== undefined) { this.warnDroppedReservedField(method, 'service', 'nothing — fixed at add() time') @@ -1750,6 +1763,10 @@ export class Brainy implements BrainyInterface { typeof reserved.weight === 'number' && { weight: reserved.weight }), ...(params.subtype === undefined && typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), + ...(params.visibility === undefined && + (reserved.visibility === 'public' || reserved.visibility === 'internal') && { + visibility: reserved.visibility as 'public' | 'internal' + }), ...(params.service === undefined && typeof reserved.service === 'string' && { service: reserved.service }), ...(params.createdBy === undefined && @@ -1828,6 +1845,11 @@ export class Brainy implements BrainyInterface { if (reserved._rev !== undefined) { this.warnDroppedReservedField(method, '_rev', 'nothing — system-managed') } + // 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal' + // value IS user-settable and is silently remapped to the top-level param below.) + if (reserved.visibility === 'system') { + this.warnDroppedReservedField(method, 'visibility', "the 'visibility' param ('public' | 'internal')") + } if (method === 'updateRelation' && reserved.service !== undefined) { this.warnDroppedReservedField(method, 'service', 'nothing — fixed at relate() time') } @@ -1861,6 +1883,10 @@ export class Brainy implements BrainyInterface { typeof reserved.weight === 'number' && { weight: reserved.weight }), ...(params.subtype === undefined && typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), + ...(params.visibility === undefined && + (reserved.visibility === 'public' || reserved.visibility === 'internal') && { + visibility: reserved.visibility as 'public' | 'internal' + }), ...(params.service === undefined && typeof reserved.service === 'string' && { service: reserved.service }) } @@ -1894,7 +1920,11 @@ export class Brainy implements BrainyInterface { ...(params.weight === undefined && typeof reserved.weight === 'number' && { weight: reserved.weight }), ...(params.subtype === undefined && - typeof reserved.subtype === 'string' && { subtype: reserved.subtype }) + typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), + ...(params.visibility === undefined && + (reserved.visibility === 'public' || reserved.visibility === 'internal') && { + visibility: reserved.visibility as 'public' | 'internal' + }) } } @@ -2043,7 +2073,13 @@ export class Brainy implements BrainyInterface { ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), // Update subtype if provided, otherwise preserve existing ...(params.subtype !== undefined && { subtype: params.subtype }), - ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }) + ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), + // Visibility: take the new value if provided, else preserve existing. Stored only + // when the effective value is not 'public' (absent === public, keeps records lean). + // A change to 'public' therefore drops the field entirely. + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }) } // Build entity structure for metadata index (with top-level fields) @@ -2054,6 +2090,9 @@ export class Brainy implements BrainyInterface { level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }), confidence: params.confidence !== undefined ? params.confidence : existing.confidence, weight: params.weight !== undefined ? params.weight : existing.weight, createdAt: existing.createdAt, @@ -2539,6 +2578,9 @@ export class Brainy implements BrainyInterface { ...(params.metadata || {}), verb: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.service !== undefined && { service: params.service }), @@ -2555,6 +2597,8 @@ export class Brainy implements BrainyInterface { verb: params.type, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.service !== undefined && { service: params.service }), @@ -2751,6 +2795,11 @@ export class Brainy implements BrainyInterface { ...(params.subtype !== undefined ? { subtype: params.subtype } : existingRec.subtype !== undefined && { subtype: existingRec.subtype }), + // Visibility: new value if provided, else preserve existing; stored only when the + // effective value is not 'public' (a change to 'public' drops the field). + ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existingRec.visibility + }), weight: params.weight ?? existingRec.weight ?? 1.0, ...(params.confidence !== undefined ? { confidence: params.confidence } @@ -2777,6 +2826,9 @@ export class Brainy implements BrainyInterface { ...(params.subtype !== undefined ? { subtype: params.subtype } : existingRec.subtype !== undefined && { subtype: existingRec.subtype }), + ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existingRec.visibility + }), weight: updatedMetadata.weight, metadata: newMetadata, data: updatedMetadata.data, @@ -2888,6 +2940,14 @@ export class Brainy implements BrainyInterface { filter.service = params.service } + // Visibility (8.0): exclude hidden tiers by default. The exclusion is applied in the + // storage scan AFTER metadata load (where verb.visibility is known), so the storage + // limit stays exact. Like subtype, it disqualifies the metadata-less fast paths. + const excludedTiers = this.excludedVisibilityTiers(params) + if (excludedTiers) { + filter.excludeVisibility = excludedTiers + } + // VFS relationships are no longer filtered // VFS is part of the knowledge graph - users can filter explicitly if needed @@ -3538,6 +3598,46 @@ export class Brainy implements BrainyInterface { }) } + /** + * The visibility tiers a read should EXCLUDE, given the caller's opt-ins. + * Default (no opts) hides both `'internal'` and `'system'`; `includeInternal` + * unhides internal; `includeSystem` unhides system. Returns `null` when nothing + * is excluded (both opts set) so callers can skip the filter entirely. + * + * @param params - The read params carrying `includeInternal` / `includeSystem`. + * @returns The set of excluded tier strings, or `null` for "exclude nothing". + */ + private excludedVisibilityTiers( + params: { includeInternal?: boolean; includeSystem?: boolean } + ): EntityVisibility[] | null { + const excluded: EntityVisibility[] = [] + if (!params.includeInternal) excluded.push('internal') + if (!params.includeSystem) excluded.push('system') + return excluded.length > 0 ? excluded : null + } + + /** + * Resolve the set of entity/relationship ids to hide for this read, by asking + * the metadata index for everything tagged with an excluded visibility tier. + * Public entities carry NO stored `visibility` (absent === public), so they are + * never in this set — exactly the entities we want to keep. Returns an empty set + * when nothing is excluded. The result is used as a HARD candidate filter + * (subtracted from candidate ids BEFORE pagination), so `limit`/`offset` stay correct. + * + * @param params - The read params carrying the visibility opt-ins. + * @returns A set of ids to omit from results. + */ + private async resolveHiddenIds( + params: { includeInternal?: boolean; includeSystem?: boolean } + ): Promise> { + const excluded = this.excludedVisibilityTiers(params) + if (!excluded) return new Set() + const ids = await this.metadataIndex.getIdsForFilter({ + visibility: excluded.length === 1 ? excluded[0] : { oneOf: excluded } + }) + return new Set(ids) + } + async find(query: string | FindParams): Promise[]> { await this.ensureInitialized() @@ -3557,6 +3657,13 @@ export class Brainy implements BrainyInterface { return this.findAggregate(params) } + // Visibility (8.0): resolve the ids to hide for this read ONCE. Default hides + // internal + system; opt back in with includeInternal / includeSystem. Applied as a + // hard candidate filter at every candidate-gathering point below so limit/offset stay + // correct. Empty set when both opts are set (nothing excluded). + const hiddenIds = await this.resolveHiddenIds(params) + const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id) + const startTime = Date.now() const result = await (async () => { let results: Result[] = [] @@ -3632,6 +3739,9 @@ export class Brainy implements BrainyInterface { filteredIds = await this.metadataIndex.getIdsForFilter(filter) } + // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact. + if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) + // Paginate BEFORE loading entities (production-scale!) const limit = params.limit || 10 const offset = params.offset || 0 @@ -3658,7 +3768,8 @@ export class Brainy implements BrainyInterface { // Unfiltered sort: column store handles this at O(K log S) scale. // No per-entity storage reads, no bucketing precision loss. if (params.orderBy) { - const k = limit + offset + // Over-fetch by the hidden count so dropping hidden ids still fills the page. + const k = limit + offset + hiddenIds.size const sortedIntIds = await this.metadataIndex.columnStore.sortTopK( params.orderBy, params.order || 'asc', @@ -3668,9 +3779,11 @@ export class Brainy implements BrainyInterface { // Convert int IDs (BigInt at the provider boundary) to UUIDs and // paginate. Number() narrowing is lossless under the u32 guard. const idMapper = this.metadataIndex.getIdMapper() - const allUuids = sortedIntIds + let allUuids = sortedIntIds .map(intId => idMapper.getUuid(Number(intId))) .filter((uuid): uuid is string => uuid !== undefined) + // Visibility hard filter — drop hidden ids BEFORE pagination. + if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) const entitiesMap = await this.batchGet(pageIds) @@ -3691,9 +3804,13 @@ export class Brainy implements BrainyInterface { filter.isVFSEntity = { ne: true } } - // Use metadata index if we need to filter + // Use metadata index when there's an actual filter (e.g. excludeVFS). The empty + // filter returns nothing from getIdsForFilter, so the unfiltered case below uses + // getNouns instead (it returns all nouns, including their visibility). if (Object.keys(filter).length > 0) { - const filteredIds = await this.metadataIndex.getIdsForFilter(filter) + let filteredIds = await this.metadataIndex.getIdsForFilter(filter) + // Visibility hard filter — drop hidden ids BEFORE pagination. + if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance @@ -3705,13 +3822,19 @@ export class Brainy implements BrainyInterface { } } } else { - // No filtering needed, use direct storage query + // No metadata filter, use direct storage query. Over-fetch by the hidden count + // so the visibility filter below still fills the requested page (limit-exact). const storageResults = await this.storage.getNouns({ - pagination: { limit: limit + offset, offset: 0 } + pagination: { limit: limit + offset + hiddenIds.size, offset: 0 } }) - for (let i = offset; i < Math.min(offset + limit, storageResults.items.length); i++) { - const noun = storageResults.items[i] + // Visibility hard filter on the materialized nouns (each carries `visibility`). + const visible = hiddenIds.size > 0 + ? storageResults.items.filter((noun) => noun && !hiddenIds.has(noun.id)) + : storageResults.items + + for (let i = offset; i < Math.min(offset + limit, visible.length); i++) { + const noun = visible[i] if (noun) { const entity = await this.convertNounToEntity(noun) results.push(this.createResult(noun.id, 1.0, entity)) @@ -3766,6 +3889,11 @@ export class Brainy implements BrainyInterface { } preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter) + // Visibility hard filter — restrict the HNSW candidate set to non-hidden ids. + if (hiddenIds.size > 0) { + preResolvedMetadataIds = preResolvedMetadataIds.filter((id) => !hiddenIds.has(id)) + } + // Short-circuit: if metadata filter matches nothing, skip expensive vector search if (preResolvedMetadataIds.length === 0) { return [] @@ -3829,6 +3957,13 @@ export class Brainy implements BrainyInterface { results = Array.from(uniqueResults.values()) } + // Visibility hard filter for vector/text/proximity candidates (the pre-resolved + // path already excluded hidden ids; this covers searches with no metadata filter). + // Applied BEFORE the final sort+slice so limit/offset remain exact. + if (hiddenIds.size > 0 && results.length > 0) { + results = results.filter((r) => !isHidden(r.id)) + } + // Apply metadata filtering using pre-resolved IDs (metadata-first optimization). // When vector search was performed, HNSW already filtered by candidateIds — // this block handles pagination, entity loading, and metadata-only queries. @@ -5443,6 +5578,7 @@ export class Brainy implements BrainyInterface { to: core.targetId, type: (reserved.verb ?? core.verb) as VerbType, ...(reserved.subtype !== undefined && { subtype: reserved.subtype as string }), + ...(reserved.visibility !== undefined && { visibility: reserved.visibility as EntityVisibility }), weight: (reserved.weight as number) ?? 1.0, data: reserved.data, metadata: custom as T, @@ -5754,6 +5890,9 @@ export class Brainy implements BrainyInterface { data: params.data, noun: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), service: params.service, createdAt: now, updatedAt: now, @@ -5769,6 +5908,8 @@ export class Brainy implements BrainyInterface { level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), createdAt: now, @@ -5871,7 +6012,12 @@ export class Brainy implements BrainyInterface { existing.weight !== undefined && { weight: existing.weight }), ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.subtype === undefined && - existing.subtype !== undefined && { subtype: existing.subtype }) + existing.subtype !== undefined && { subtype: existing.subtype }), + // Visibility: new value if provided, else preserve existing; stored only when the + // effective value is not 'public' (a change to 'public' drops the field). + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }) } const entityForIndexing = { id: params.id, @@ -5880,6 +6026,9 @@ export class Brainy implements BrainyInterface { level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }), confidence: params.confidence !== undefined ? params.confidence : existing.confidence, weight: params.weight !== undefined ? params.weight : existing.weight, createdAt: existing.createdAt, @@ -6083,6 +6232,9 @@ export class Brainy implements BrainyInterface { ...(params.metadata || {}), verb: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.service !== undefined && { service: params.service }), @@ -6097,6 +6249,8 @@ export class Brainy implements BrainyInterface { verb: params.type, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.service !== undefined && { service: params.service }), @@ -10007,6 +10161,7 @@ export class Brainy implements BrainyInterface { to: v.targetId, type: (v.verb || v.type) as VerbType, ...(v.subtype !== undefined && { subtype: v.subtype }), + ...(v.visibility !== undefined && { visibility: v.visibility }), weight: v.weight ?? 1.0, ...(typeof v.confidence === 'number' && { confidence: v.confidence }), data: v.data, diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 7c6b63ea..69133b1d 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -189,6 +189,25 @@ export interface VerbMetadata { [key: string]: unknown } +/** + * Visibility tier for an entity or relationship — controls whether it surfaces + * on Brainy's default user-facing read paths. A reserved, top-level field; the + * absence of the field is exactly equivalent to `'public'`. + * + * - `'public'` (DEFAULT, and the meaning when the field is absent) — counted and + * returned everywhere: `find()`, `related()`, `getNounCount()`/`getVerbCount()`, + * `stats()`. + * - `'internal'` — a consumer's app-internal data. HIDDEN from the default + * `find()` / `related()` / counts / `stats()`, but retrievable with an explicit + * opt-in (`find({ includeInternal: true })`, `related({ includeInternal: true })`). + * - `'system'` — Brainy's own plumbing (e.g. the VFS root entity). Hidden + * EVERYWHERE by default and surfaced only via the explicit `includeSystem` + * opt-in. NOT settable by consumer code — only internal Brainy code assigns it, + * which is why the `add()` / `relate()` params type narrows to + * `'public' | 'internal'`. + */ +export type EntityVisibility = 'public' | 'internal' | 'system' + /** * Combined noun structure for transport/API boundaries * @@ -218,6 +237,14 @@ export interface HNSWNounWithMetadata { // fast path, never falling through to the metadata fallback. subtype?: string + // VISIBILITY — three-value tier gating whether this entity surfaces on default + // user-facing reads. Absent === 'public' (counted + returned everywhere). 'internal' + // hides the entity from default find()/counts/stats but keeps it retrievable via + // find({ includeInternal: true }). 'system' (Brainy plumbing, e.g. the VFS root) is + // hidden everywhere unless find({ includeSystem: true }). Stored only when not 'public' + // so the common case stays lean. See {@link EntityVisibility}. + visibility?: EntityVisibility + // QUALITY METRICS (top-level, explicit) confidence?: number weight?: number @@ -260,6 +287,7 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet = new Set([ 'level', 'type', 'subtype', + 'visibility', 'confidence', 'weight', 'createdAt', @@ -319,6 +347,7 @@ export const STANDARD_VERB_FIELDS: ReadonlySet = new Set([ 'sourceId', 'targetId', 'subtype', + 'visibility', 'confidence', 'weight', 'createdAt', @@ -390,6 +419,13 @@ export interface HNSWVerbWithMetadata { // the metadata fallback. subtype?: string + // VISIBILITY — verb mirror of the entity tier. Absent === 'public' (counted + + // returned everywhere). 'internal' hides the edge from default related()/counts/stats + // but keeps it retrievable via related({ includeInternal: true }). 'system' is hidden + // everywhere unless related({ includeSystem: true }). Stored only when not 'public'. + // See {@link EntityVisibility}. + visibility?: EntityVisibility + // QUALITY METRICS (top-level, explicit) weight?: number confidence?: number @@ -423,6 +459,7 @@ export interface GraphVerb { connections?: Map> // Optional connections from the vector index type?: string // Optional verb type subtype?: string // Optional sub-classification within the VerbType (7.30+) + visibility?: EntityVisibility // Visibility tier (8.0); absent === 'public'. See {@link EntityVisibility}. weight?: number // Optional weight of the relationship confidence?: number // Optional confidence score (0-1) metadata?: any // Optional metadata for the verb diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 53dcd901..1deb6746 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -16,6 +16,7 @@ import { HNSWVerbWithMetadata, StatisticsData } from '../coreTypes.js' +import type { EntityVisibility } from '../coreTypes.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' import { validateNounType, validateVerbType } from '../utils/typeValidation.js' import { @@ -192,6 +193,18 @@ function getVerbMetadataPath(id: string): string { return `entities/verbs/${shard}/${id}/metadata.json` } +/** + * Extract the entity id from an ID-first metadata path. Both noun and verb + * metadata live at `entities////metadata.json`, so the id is + * the second-to-last path segment. + * @param path - A metadata.json path produced by `getNounMetadataPath` / `getVerbMetadataPath`. + * @returns The id segment, or `undefined` if the path doesn't have the expected shape. + */ +function idFromMetadataPath(path: string): string | undefined { + const segments = path.split('/') + return segments[segments.length - 2] +} + /** * Optional count capabilities probed via duck typing by getNouns()/getVerbs(). * Adapters with a native O(1) count API may implement these; they are not part @@ -213,6 +226,19 @@ interface OptionalCountCapabilities { }) => Promise } +/** + * Whether an entity/relationship with the given visibility tier counts toward + * the user-facing counts (`counts.json` totals, `nounCountsByType`, `stats()`). + * Public entities (absent tier === `'public'`) are counted; `'internal'` and + * `'system'` are excluded. Single source of truth for the count-exclusion rule. + * + * @param visibility - The stored visibility value (may be `undefined`/`unknown`). + * @returns `true` when the entity should be counted, `false` for internal/system. + */ +function isCountedVisibility(visibility: unknown): boolean { + return visibility !== 'internal' && visibility !== 'system' +} + /** * Base storage adapter that implements common functionality * This is an abstract class that should be extended by specific storage adapters @@ -313,6 +339,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected verbSubtypeByIdCache = new Map() // Total: 676 bytes (99.2% reduction vs Map-based tracking) + /** + * In-memory map from noun id → its non-public `visibility` tier ('internal' | + * 'system'). Parallel to `nounTypeByIdCache`. Lets `saveNoun_internal()` + * (which has no metadata access in its signature) skip the user-facing + * `nounCountsByType` increment for hidden entities, and lets + * `deleteNounMetadata()` skip the matching decrement — keeping the counts + * symmetric. Sparse by design: public entities (the common case) get NO + * entry, so the absence of an id means "public, counted". + */ + protected nounVisibilityByIdCache = new Map() + + /** + * In-memory map from verb id → its non-public `visibility` tier. Verb-side + * mirror of `nounVisibilityByIdCache`. Sparse — public edges get no entry. + */ + protected verbVisibilityByIdCache = new Map() + // Type caches REMOVED - ID-first paths eliminate need for type lookups! // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json // Type is just a field in the metadata, indexed by MetadataIndexManager for queries @@ -1652,6 +1695,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { : [subtypeFilterValue] ) : null + // 8.0 visibility exclusion — applied after metadata load (verb visibility lives in + // metadata), so hidden edges are skipped BEFORE the pagination window fills. + const excludeVisibility = (filter as { excludeVisibility?: string[] } | undefined)?.excludeVisibility + const filterExcludeVisibility = excludeVisibility && excludeVisibility.length > 0 + ? new Set(excludeVisibility) + : null // Iterate by shards (0x00-0xFF) instead of types - single pass! for (let shard = 0; shard < 256 && collectedVerbs.length < peekCount; shard++) { @@ -1698,6 +1747,15 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } + // Apply visibility exclusion (8.0). Absent === 'public' (kept); a stored + // 'internal'/'system' value is dropped when its tier is excluded. + if (filterExcludeVisibility) { + const visibility = metadata?.visibility as string | undefined + if (visibility && filterExcludeVisibility.has(visibility)) { + continue + } + } + // Combine verb + metadata via the canonical hydration helper — // reserved fields top-level, ONLY custom fields in `metadata` // (this site previously echoed the full flat record). @@ -1744,6 +1802,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId?: string | string[] service?: string | string[] metadata?: Record + /** + * 8.0 visibility: tiers to exclude from results (e.g. `['internal','system']`). + * Applied after metadata load in the full scan, so it disqualifies the + * metadata-less graph-index fast paths (same as `subtype`). + */ + excludeVisibility?: string[] } }): Promise<{ items: HNSWVerbWithMetadata[] @@ -1764,10 +1828,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { // fallthrough to the full shard scan (which loads metadata and applies the // subtype check after the load). Subtype is not stored on the raw HNSWVerb; // it's a metadata field. The graph-index fast paths return verbs without - // metadata, so they can't apply subtype filtering correctly. + // metadata, so they can't apply subtype filtering correctly. The 8.0 + // `excludeVisibility` filter (also a metadata field) disqualifies them the same way. if ( options?.filter && - !(options.filter as { verbType?: string | string[]; subtype?: string | string[] }).subtype + !(options.filter as { verbType?: string | string[]; subtype?: string | string[] }).subtype && + !(options.filter as { excludeVisibility?: string[] }).excludeVisibility ) { // CRITICAL VFS FIX: If filtering by sourceId + verbType (most common VFS pattern!) // This is the query PathResolver.getChildren() uses: related({ from: dirId, type: VerbType.Contains }) @@ -2367,16 +2433,40 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.nounSubtypeByIdCache.delete(id) } + // Visibility (8.0): only public entities count toward the user-facing totals. + // Warm `nounVisibilityByIdCache` so `saveNoun_internal()` (no metadata access) + // can gate the `nounCountsByType` increment, and `deleteNounMetadata()` can gate + // the matching decrement. Sparse — public ids get no entry. + const newVisibility = metadata.visibility + const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) + const isCounted = isCountedVisibility(newVisibility) + if (isCountedVisibility(newVisibility)) { + this.nounVisibilityByIdCache.delete(id) + } else { + this.nounVisibilityByIdCache.set(id, newVisibility as EntityVisibility) + } + // CRITICAL FIX: Increment count for new entities // This runs AFTER metadata is saved, guaranteeing type information is available // Uses synchronous increment since storage operations are already serialized // Fixes Bug #1: Count synchronization failure during add() and import() - if (isNew && metadata.noun) { + // 8.0: skip the user-facing total for internal/system entities (counts.json + getNounCount()). + if (isNew && metadata.noun && isCounted) { this.incrementEntityCount(metadata.noun) // Persist counts asynchronously (fire and forget) this.scheduleCountPersist().catch(() => { // Ignore persist errors - will retry on next operation }) + } else if (!isNew && metadata.noun && wasCounted !== isCounted) { + // Visibility flipped on update(): move the entity in/out of the user-facing total + // (counts.json / getNounCount()). `nounCountsByType` is gated independently in + // `saveNoun_internal()` off the cache warmed just above, so it is not touched here. + if (isCounted) { + this.incrementEntityCount(metadata.noun) + } else { + this.decrementEntityCount(metadata.noun) + } + this.scheduleCountPersist().catch(() => {}) } // 8.0 MVCC: entity-visible write — advance the generation watermark @@ -2707,11 +2797,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { // decrement `nounCountsByType` so type-statistics stay honest across // deletes; symmetric with the increment in `saveNounMetadata_internal()`. const priorType = this.nounTypeByIdCache.get(id) + // 8.0 visibility: an internal/system entity was never added to `nounCountsByType` + // (gated in `saveNoun_internal()`), so it must not be decremented here either. + const priorCounted = isCountedVisibility(this.nounVisibilityByIdCache.get(id)) + this.nounVisibilityByIdCache.delete(id) if (priorType) { this.nounTypeByIdCache.delete(id) - const idx = TypeUtils.getNounIndex(priorType) - if (this.nounCountsByType[idx] > 0) { - this.nounCountsByType[idx]-- + if (priorCounted) { + const idx = TypeUtils.getNounIndex(priorType) + if (this.nounCountsByType[idx] > 0) { + this.nounCountsByType[idx]-- + } } // Symmetric subtype decrement @@ -2803,16 +2899,52 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.verbSubtypeByIdCache.delete(id) } + // Visibility (8.0): verb mirror of the noun count gating. Warm + // `verbVisibilityByIdCache` so `deleteVerbMetadata()` can prune it. Sparse — + // public edges get no entry. + // + // NOTE on `verbCountsByType`: unlike the noun path, `saveVerb_internal()` runs + // BEFORE this method (relate() saves the verb vector first) and has already done + // an UNCONDITIONAL `verbCountsByType[idx]++`. We therefore COMPENSATE here: for a + // new hidden edge, undo that bump. `updateRelation()` does not re-run + // `saveVerb_internal()`, so on a visibility flip we adjust the bucket directly. + const newVisibility = metadata.visibility + const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) + const isCounted = isCountedVisibility(newVisibility) + if (isCounted) { + this.verbVisibilityByIdCache.delete(id) + } else { + this.verbVisibilityByIdCache.set(id, newVisibility as EntityVisibility) + } + const verbTypeIdx = TypeUtils.getVerbIndex(verbType) + // CRITICAL FIX: Increment verb count for new relationships // This runs AFTER metadata is saved // Uses synchronous increment since storage operations are already serialized // Fixes Bug #2: Count synchronization failure during relate() and import() + // 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()). if (isNew) { - this.incrementVerbCount(verbType) + if (isCounted) { + this.incrementVerbCount(verbType) + } else { + // Hidden edge: undo the unconditional bump from saveVerb_internal(). + if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]-- + } // Persist counts asynchronously (fire and forget) this.scheduleCountPersist().catch(() => { // Ignore persist errors - will retry on next operation }) + } else if (wasCounted !== isCounted) { + // Visibility flipped on updateRelation() (saveVerb_internal did not run): move the + // edge in/out of both the user-facing total and the per-type bucket. + if (isCounted) { + this.incrementVerbCount(verbType) + this.verbCountsByType[verbTypeIdx]++ + } else { + this.decrementVerbCount(verbType) + if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]-- + } + this.scheduleCountPersist().catch(() => {}) } // 8.0 MVCC: entity-visible write — advance the generation watermark @@ -2855,6 +2987,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.verbSubtypeByIdCache.delete(id) this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype) } + // 8.0 visibility: prune the cache entry (verb deletes don't touch verbCountsByType + // in this path, mirroring the existing verb-subtype delete semantics). + this.verbVisibilityByIdCache.delete(id) // 8.0 MVCC: entity-visible write — advance the generation watermark // (suppressed inside transact batches by the generation store). @@ -3290,9 +3425,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { const metadata = await this.readCanonicalObject(path) if (metadata && metadata.noun) { - const typeIndex = TypeUtils.getNounIndex(metadata.noun) - if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { - this.nounCountsByType[typeIndex]++ + // 8.0 visibility: rebuild only public entities into the user-facing + // per-type stat; warm the cache so later writes/deletes stay symmetric. + const id = idFromMetadataPath(path) + if (isCountedVisibility(metadata.visibility)) { + const typeIndex = TypeUtils.getNounIndex(metadata.noun) + if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { + this.nounCountsByType[typeIndex]++ + } + if (id) this.nounVisibilityByIdCache.delete(id) + } else if (id) { + this.nounVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) } } } catch (error) { @@ -3318,9 +3461,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { const metadata = await this.readCanonicalObject(path) if (metadata && metadata.verb) { - const typeIndex = TypeUtils.getVerbIndex(metadata.verb) - if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { - this.verbCountsByType[typeIndex]++ + // 8.0 visibility: rebuild only public edges into the user-facing per-type stat. + const id = idFromMetadataPath(path) + if (isCountedVisibility(metadata.visibility)) { + const typeIndex = TypeUtils.getVerbIndex(metadata.verb) + if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { + this.verbCountsByType[typeIndex]++ + } + if (id) this.verbVisibilityByIdCache.delete(id) + } else if (id) { + this.verbVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) } } } catch (error) { @@ -3497,17 +3647,24 @@ export abstract class BaseStorage extends BaseStorageAdapter { const type = this.getNounType(noun) const path = getNounVectorPath(noun.id) - // Update type tracking + // Update type tracking — but only for entities that count toward the + // user-facing per-type stats. `nounVisibilityByIdCache` (warmed by + // `saveNounMetadata_internal()`) flags internal/system ids; public ids are + // absent, so the common case still increments. 8.0 visibility exclusion. const typeIndex = TypeUtils.getNounIndex(type) - this.nounCountsByType[typeIndex]++ + const counted = isCountedVisibility(this.nounVisibilityByIdCache.get(noun.id)) + if (counted) { + this.nounCountsByType[typeIndex]++ + } // Write-cache coherent canonical write await this.writeCanonicalObject(path, noun) // Periodically save statistics // Also save on first noun of each type to ensure low-count types are tracked - const shouldSave = this.nounCountsByType[typeIndex] === 1 || // First noun of type - this.nounCountsByType[typeIndex] % 100 === 0 // Every 100th + const shouldSave = counted && + (this.nounCountsByType[typeIndex] === 1 || // First noun of type + this.nounCountsByType[typeIndex] % 100 === 0) // Every 100th if (shouldSave) { await this.saveTypeStatistics() } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index b0ba5978..5f916f5a 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -5,6 +5,7 @@ */ import { StorageAdapter, Vector } from '../coreTypes.js' +import type { EntityVisibility } from '../coreTypes.js' import { NounType, VerbType } from './graphTypes.js' import type { EntityMetadataInput, @@ -39,6 +40,13 @@ export interface Entity { * rolled into per-NounType statistics. */ subtype?: string + /** + * Visibility tier (reserved top-level field). Absent === `'public'` (counted and + * returned everywhere). `'internal'` hides the entity from default `find()` / counts / + * `stats()` (opt back in with `find({ includeInternal: true })`); `'system'` is Brainy + * plumbing, hidden unless `find({ includeSystem: true })`. See {@link EntityVisibility}. + */ + visibility?: EntityVisibility /** Opaque content — used for embeddings and semantic search. Not indexed by MetadataIndex. */ data?: any /** User-defined structured fields — indexed and queryable via `where` filters. */ @@ -90,6 +98,14 @@ export interface Relation { * the column-store directly. */ subtype?: string + /** + * Visibility tier (reserved top-level field), the verb mirror of `Entity.visibility`. + * Absent === `'public'` (counted and returned everywhere). `'internal'` hides the edge + * from default `related()` / counts / `stats()` (opt back in with + * `related({ includeInternal: true })`); `'system'` is Brainy plumbing. See + * {@link EntityVisibility}. + */ + visibility?: EntityVisibility /** Connection strength (0-1, default: 1.0) */ weight?: number /** Opaque content for the relationship (overrides auto-computed vector if provided) */ @@ -135,6 +151,7 @@ export interface Result { // Convenience: Common entity fields flattened to top level type?: NounType // Entity type (from entity.type) subtype?: string // Per-product sub-classification (from entity.subtype) + visibility?: EntityVisibility // Visibility tier (from entity.visibility); absent === 'public' metadata?: T // Entity metadata (from entity.metadata) data?: any // Entity data (from entity.data) confidence?: number // Type classification confidence (from entity.confidence) @@ -289,16 +306,25 @@ export interface AddParams { * aggregation on the standard-field fast path. */ subtype?: string + /** + * Visibility tier (reserved top-level field). Omit (or pass `'public'`) for normal + * data that is counted and returned everywhere. Pass `'internal'` for app-internal + * data that should be hidden from default `find()` / counts / `stats()` but stay + * retrievable via `find({ includeInternal: true })`. The `'system'` tier is reserved + * for Brainy's own plumbing and is intentionally not accepted here. Stored only when + * not `'public'`. + */ + visibility?: 'public' | 'internal' /** * Structured queryable fields — indexed by MetadataIndex, used in `where` filters. * - * Reserved entity fields (`RESERVED_ENTITY_FIELDS` — `noun`, `subtype`, `createdAt`, - * `updatedAt`, `confidence`, `weight`, `service`, `data`, `createdBy`, `_rev`) may NOT - * appear here — they have dedicated top-level params and the type makes a literal - * reserved key a compile error. Untyped (JavaScript) callers that pass one anyway are - * normalized at write time: user-settable fields remap to their top-level param - * (top-level wins when both are supplied), system-managed fields are dropped with a - * one-shot warning. + * Reserved entity fields (`RESERVED_ENTITY_FIELDS` — `noun`, `subtype`, `visibility`, + * `createdAt`, `updatedAt`, `confidence`, `weight`, `service`, `data`, `createdBy`, + * `_rev`) may NOT appear here — they have dedicated top-level params and the type makes + * a literal reserved key a compile error. Untyped (JavaScript) callers that pass one + * anyway are normalized at write time: user-settable fields remap to their top-level + * param (top-level wins when both are supplied), system-managed fields are dropped with + * a one-shot warning. */ metadata?: EntityMetadataInput /** Custom entity ID (auto-generated UUID v4 if not provided) */ @@ -330,11 +356,18 @@ export interface UpdateParams { data?: any // New content to re-embed type?: NounType // Change type subtype?: string // Change subtype (set to '' or null-equivalent via dedicated unset is future work) + /** + * Change the visibility tier. Omit to preserve the existing value. Toggling between + * `'public'` and `'internal'` moves the entity in/out of the default-visible counts and + * `find()` results. The `'system'` tier is Brainy-internal (e.g. re-asserted on the VFS + * root during repair) — consumer code should only ever use `'public'` / `'internal'`. + */ + visibility?: EntityVisibility /** * Metadata fields to merge (or replace when `merge: false`). Reserved entity * fields (`RESERVED_ENTITY_FIELDS`) may NOT appear here — `confidence` / - * `weight` / `subtype` have dedicated params on this call, and the rest are - * system-managed. A literal reserved key is a compile error; untyped callers + * `weight` / `subtype` / `visibility` have dedicated params on this call, and the rest + * are system-managed. A literal reserved key is a compile error; untyped callers * are normalized at write time (remap user-settable, drop system-managed * with a one-shot warning). */ @@ -375,14 +408,23 @@ export interface RelateParams { * (`related({ verb, subtype })`) and aggregation (`groupBy:['subtype']`). */ subtype?: string + /** + * Visibility tier (reserved top-level field), the verb mirror of `AddParams.visibility`. + * Omit (or pass `'public'`) for normal edges that are counted and returned everywhere. + * Pass `'internal'` for app-internal edges hidden from default `related()` / counts / + * `stats()` but retrievable via `related({ includeInternal: true })`. The `'system'` + * tier is reserved for Brainy's own plumbing and is not accepted here. Stored only when + * not `'public'`. + */ + visibility?: 'public' | 'internal' /** Connection strength (0-1, default: 1.0) */ weight?: number /** Content for the relationship (optional — overrides auto-computed vector) */ data?: any /** * Structured queryable fields on the edge. Reserved relationship fields - * (`RESERVED_RELATION_FIELDS` — `verb`, `subtype`, `createdAt`, `updatedAt`, - * `confidence`, `weight`, `service`, `data`, `createdBy`, `_rev`) may NOT + * (`RESERVED_RELATION_FIELDS` — `verb`, `subtype`, `visibility`, `createdAt`, + * `updatedAt`, `confidence`, `weight`, `service`, `data`, `createdBy`, `_rev`) may NOT * appear here — they have dedicated params. A literal reserved key is a * compile error; untyped callers are normalized at write time. */ @@ -404,6 +446,12 @@ export interface UpdateRelationParams { id: string // Relation to update type?: VerbType // Change verb type subtype?: string // Change sub-classification (omit to preserve existing) + /** + * Change the visibility tier. Omit to preserve the existing value. The verb mirror of + * `UpdateParams.visibility`; `'system'` is Brainy-internal — consumer code should only + * use `'public'` / `'internal'`. + */ + visibility?: EntityVisibility weight?: number // New weight confidence?: number // New confidence (0-1) data?: any // New content @@ -448,7 +496,24 @@ export interface FindParams { subtype?: string | string[] /** Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`) */ where?: Partial - + + // Visibility + /** + * Also return `visibility: 'internal'` entities. By default `find()` returns ONLY + * public entities (those with no `visibility` field, or `visibility: 'public'`); + * app-internal entities are hidden. Set `true` to include them. Applied as a hard + * candidate filter, so `limit` / `offset` stay correct. Does NOT include `'system'` + * entities — use `includeSystem` for those. + */ + includeInternal?: boolean + /** + * Also return `visibility: 'system'` entities (Brainy's own plumbing, e.g. the VFS + * root). Hidden by default and even when `includeInternal` is set. Set `true` only + * when you specifically need to see system entities. Applied as a hard candidate + * filter, so `limit` / `offset` stay correct. + */ + includeSystem?: boolean + // Graph Intelligence connected?: GraphConstraints @@ -609,6 +674,25 @@ export interface RelatedParams { */ subtype?: string | string[] + /** + * Also return `visibility: 'internal'` relationships. + * + * By default `related()` returns ONLY public relationships (those with no + * `visibility` field, or `visibility: 'public'`); app-internal edges are hidden. + * Set `true` to include them. Applied as a hard candidate filter so `limit` / + * `offset` stay correct. Does NOT include `'system'` edges — use `includeSystem`. + */ + includeInternal?: boolean + + /** + * Also return `visibility: 'system'` relationships (Brainy's own plumbing). + * + * Hidden by default and even when `includeInternal` is set. Set `true` only when + * you specifically need system edges. Applied as a hard candidate filter so + * `limit` / `offset` stay correct. + */ + includeSystem?: boolean + /** * Maximum number of results to return * diff --git a/src/types/reservedFields.ts b/src/types/reservedFields.ts index a67b0a5c..a0606e1d 100644 --- a/src/types/reservedFields.ts +++ b/src/types/reservedFields.ts @@ -35,6 +35,7 @@ * |-----|----------------------| * | `noun` | the `type` param of `add()` / `update()` (stored under the key `noun`) | * | `subtype` | the `subtype` param | + * | `visibility` | the `visibility` param (`'public'` \| `'internal'`; `'system'` is Brainy-only) | * | `createdAt` | system-managed — set once at `add()` time | * | `updatedAt` | system-managed — set on every write | * | `confidence` | the `confidence` param | @@ -52,6 +53,7 @@ export const RESERVED_ENTITY_FIELDS = [ 'noun', 'subtype', + 'visibility', 'createdAt', 'updatedAt', 'confidence', @@ -78,6 +80,7 @@ export type ReservedEntityField = (typeof RESERVED_ENTITY_FIELDS)[number] * |-----|----------------------| * | `verb` | the `type` param of `relate()` / `updateRelation()` (stored under the key `verb`) | * | `subtype` | the `subtype` param | + * | `visibility` | the `visibility` param (`'public'` \| `'internal'`; `'system'` is Brainy-only) | * | `createdAt` | system-managed — set once at `relate()` time | * | `updatedAt` | system-managed — set on every write | * | `confidence` | the `confidence` param | @@ -90,6 +93,7 @@ export type ReservedEntityField = (typeof RESERVED_ENTITY_FIELDS)[number] export const RESERVED_RELATION_FIELDS = [ 'verb', 'subtype', + 'visibility', 'createdAt', 'updatedAt', 'confidence', diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 9cb3dd24..4b7e3fd6 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -549,6 +549,7 @@ export function validateUpdateParams(params: UpdateParams): void { !params.type && !params.vector && params.subtype === undefined && + params.visibility === undefined && params.confidence === undefined && params.weight === undefined ) { diff --git a/src/utils/rebuildCounts.ts b/src/utils/rebuildCounts.ts index 0e898ddb..340de352 100644 --- a/src/utils/rebuildCounts.ts +++ b/src/utils/rebuildCounts.ts @@ -116,6 +116,11 @@ export async function rebuildCounts(storage: BaseStorage): Promise