diff --git a/src/brainy.ts b/src/brainy.ts index 01ac6014..3c1a6ac0 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -12,6 +12,7 @@ import { HNSWIndex } from './hnsw/hnswIndex.js' import { createStorage } from './storage/storageFactory.js' import { BaseStorage } from './storage/baseStorage.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' +import type { EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, cosineDistance, @@ -1038,6 +1039,14 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateAddParams(params) + // Reserved 'visibility' arriving via the metadata bag is normalized to the + // top-level param, mirroring add()'s lift behavior for the other reserved + // fields. A 'public'/'internal' value is user-settable and remaps to + // params.visibility (top-level wins when both are supplied); 'system' is + // Brainy-only and is dropped with a one-shot warning. In every case the key + // is stripped from metadata so it never echoes inside the custom bag. + params = this.remapReservedVisibilityOnAdd('add', params) + // Tracked-field vocabulary enforcement (Layer 2). Walks both bags so a // tracked field declared at top level (e.g. 'subtype') and one declared in // metadata (e.g. 'status') both validate. @@ -1084,6 +1093,9 @@ export class Brainy implements BrainyInterface { data: params.data, noun: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), service: params.service, createdAt: Date.now(), updatedAt: Date.now(), @@ -1105,6 +1117,8 @@ export class Brainy implements BrainyInterface { level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), createdAt: Date.now(), @@ -1378,6 +1392,7 @@ export class Brainy implements BrainyInterface { // Flatten common entity fields to top level type: entity.type, subtype: entity.subtype, + visibility: entity.visibility, metadata: entity.metadata, data: entity.data, confidence: entity.confidence, @@ -1408,6 +1423,7 @@ export class Brainy implements BrainyInterface { vector: noun.vector, type: noun.type || NounType.Thing, subtype: noun.subtype, + visibility: noun.visibility, // Standard fields at top-level confidence: noun.confidence, @@ -1451,13 +1467,14 @@ export class Brainy implements BrainyInterface { // Extract standard fields, rest are custom metadata // Same destructuring as baseStorage.getNoun() to ensure consistency - const { noun, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata + const { noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata const entity: Entity = { id, vector: [], // Stub vector (empty array - vectors not loaded for metadata-only) type: noun as NounType || NounType.Thing, subtype, + visibility: visibility as EntityVisibility | undefined, // Standard fields from metadata confidence, @@ -1544,12 +1561,13 @@ export class Brainy implements BrainyInterface { if (!params.metadata || typeof params.metadata !== 'object') return params const { - noun, subtype, createdAt, updatedAt, confidence, weight, service, + noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = params.metadata as Record const hasReserved = - noun !== undefined || subtype !== undefined || createdAt !== undefined || + noun !== undefined || subtype !== undefined || visibility !== undefined || + createdAt !== undefined || updatedAt !== undefined || confidence !== undefined || weight !== undefined || service !== undefined || data !== undefined || createdBy !== undefined || _rev !== undefined @@ -1572,16 +1590,125 @@ export class Brainy implements BrainyInterface { if (service !== undefined) warnDropped('service', 'nothing — fixed at add() time') if (createdBy !== undefined) warnDropped('createdBy', 'nothing — fixed at add() time') if (_rev !== undefined) warnDropped('_rev', "the 'ifRev' param for optimistic concurrency") + // 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal' + // value IS user-settable and is silently remapped to the top-level param below.) + if (visibility === 'system') { + warnDropped('visibility', "the 'visibility' param ('public' | 'internal')") + } return { ...params, metadata: customMetadata as UpdateParams['metadata'], ...(params.confidence === undefined && typeof confidence === 'number' && { confidence }), ...(params.weight === undefined && typeof weight === 'number' && { weight }), - ...(params.subtype === undefined && typeof subtype === 'string' && { subtype }) + ...(params.subtype === undefined && typeof subtype === 'string' && { subtype }), + ...(params.visibility === undefined && + (visibility === 'public' || visibility === 'internal') && { + visibility: visibility as 'public' | 'internal' + }) } } + /** + * @description Normalize a reserved `visibility` field that arrived inside the + * `metadata` bag of an `add()` / `relate()` call. Untyped (JavaScript) callers + * can smuggle the reserved key past the compile-time guard; this applies the + * same contract as the rest of the reserved fields: + * - `'public'` / `'internal'` — user-settable; remapped to the top-level + * `visibility` param unless the caller also passed it explicitly (top-level + * wins), and stripped from the metadata bag. + * - `'system'` — Brainy-only; dropped with a one-shot warning and stripped from + * the metadata bag (the entity/edge stays public). + * In every case the key is removed from `metadata` so it never echoes inside the + * custom bag. + * @param method - The calling method name (`'add'` | `'relate'`) for the warning text. + * @param params - The caller's params (not mutated; a normalized copy is returned). + * @returns Params with `visibility` normalized out of `metadata`. + */ + private remapReservedVisibilityOnAdd< + P extends { visibility?: 'public' | 'internal'; metadata?: any } + >(method: 'add' | 'relate', params: P): P { + if (!params.metadata || typeof params.metadata !== 'object') return params + const bag = params.metadata as Record + if (!('visibility' in bag)) return params + + const { visibility, ...customMetadata } = bag + if (visibility === 'system') { + this.warnDroppedReservedField( + method, + 'visibility', + "the 'visibility' param ('public' | 'internal')" + ) + } + + return { + ...params, + metadata: customMetadata, + ...(params.visibility === undefined && + (visibility === 'public' || visibility === 'internal') && { + visibility: visibility as 'public' | 'internal' + }) + } + } + + /** + * @description Emit a one-shot (per field, per process) warning that a reserved + * field supplied through a metadata bag was ignored, naming the correct write + * path. Shared by the add/relate/update reserved-field normalizers. + * @param method - The calling method name, used in the message. + * @param field - The reserved field that was dropped. + * @param rightPath - Human-readable description of where the value should go instead. + */ + private warnDroppedReservedField(method: string, field: string, rightPath: string): void { + if (Brainy.warnedReservedFields.has(field)) return + Brainy.warnedReservedFields.add(field) + prodLog.warn( + `[brainy] ${method}(): '${field}' is a reserved field and cannot be set ` + + `through a metadata patch — use ${rightPath}. The value was ignored. ` + + `(This warning is shown once per field per process.)` + ) + } + + /** + * The visibility tiers a read should EXCLUDE, given the caller's opt-ins. + * Default (no opts) hides both `'internal'` and `'system'`; `includeInternal` + * unhides internal; `includeSystem` unhides system. Returns `null` when nothing + * is excluded (both opts set) so callers can skip the filter entirely. + * + * @param params - The read params carrying `includeInternal` / `includeSystem`. + * @returns The set of excluded tier strings, or `null` for "exclude nothing". + */ + private excludedVisibilityTiers( + params: { includeInternal?: boolean; includeSystem?: boolean } + ): EntityVisibility[] | null { + const excluded: EntityVisibility[] = [] + if (!params.includeInternal) excluded.push('internal') + if (!params.includeSystem) excluded.push('system') + return excluded.length > 0 ? excluded : null + } + + /** + * Resolve the set of entity/relationship ids to hide for this read, by asking + * the metadata index for everything tagged with an excluded visibility tier. + * Public entities carry NO stored `visibility` (absent === public), so they are + * never in this set — exactly the entities we want to keep. Returns an empty set + * when nothing is excluded. The result is used as a HARD candidate filter + * (subtracted from candidate ids BEFORE pagination), so `limit`/`offset` stay correct. + * + * @param params - The read params carrying the visibility opt-ins. + * @returns A set of ids to omit from results. + */ + private async resolveHiddenIds( + params: { includeInternal?: boolean; includeSystem?: boolean } + ): Promise> { + const excluded = this.excludedVisibilityTiers(params) + if (!excluded) return new Set() + const ids = await this.metadataIndex.getIdsForFilter({ + visibility: excluded.length === 1 ? excluded[0] : { oneOf: excluded } + }) + return new Set(ids) + } + /** One-shot registry for reserved-field warnings (per process). */ private static warnedReservedFields = new Set() @@ -1672,7 +1799,13 @@ export class Brainy implements BrainyInterface { ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), // Update subtype if provided, otherwise preserve existing ...(params.subtype !== undefined && { subtype: params.subtype }), - ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }) + ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), + // Visibility: take the new value if provided, else preserve existing. Stored only + // when the effective value is not 'public' (absent === public, keeps records lean). + // A change to 'public' therefore drops the field entirely. + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }) } // Build entity structure for metadata index (with top-level fields) @@ -1683,6 +1816,9 @@ export class Brainy implements BrainyInterface { level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, + ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existing.visibility + }), confidence: params.confidence !== undefined ? params.confidence : existing.confidence, weight: params.weight !== undefined ? params.weight : existing.weight, createdAt: existing.createdAt, @@ -1976,6 +2112,11 @@ export class Brainy implements BrainyInterface { // Zero-config validation (static import for performance) validateRelateParams(params) + // Reserved 'visibility' arriving via the metadata bag is normalized to the + // top-level param (mirror of add()): 'public'/'internal' lifts, 'system' is + // dropped with a one-shot warning, and the key is stripped from the bag. + params = this.remapReservedVisibilityOnAdd('relate', params) + // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered // via brain.requireSubtype() compose with the brain-wide strict-mode flag. // Metadata is passed so infrastructure edges (VFS containment) can bypass @@ -2028,6 +2169,9 @@ export class Brainy implements BrainyInterface { ...(params.metadata || {}), verb: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + // visibility: stored only when not 'public' (absent === public, keeps records lean) + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, createdAt: Date.now(), ...((params as any).data !== undefined && { data: (params as any).data }) @@ -2044,6 +2188,8 @@ export class Brainy implements BrainyInterface { verb: params.type, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), + ...(params.visibility !== undefined && + params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, metadata: params.metadata, data: (params as any).data, @@ -2213,6 +2359,11 @@ export class Brainy implements BrainyInterface { ...(params.subtype !== undefined ? { subtype: params.subtype } : existingAny.subtype !== undefined && { subtype: existingAny.subtype }), + // Visibility: new value if provided, else preserve existing; stored only when the + // effective value is not 'public' (a change to 'public' drops the field). + ...(((params.visibility ?? existingAny.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existingAny.visibility + }), weight: params.weight ?? existingAny.weight ?? 1.0, ...(params.confidence !== undefined ? { confidence: params.confidence } @@ -2237,6 +2388,9 @@ export class Brainy implements BrainyInterface { ...(params.subtype !== undefined ? { subtype: params.subtype } : existingAny.subtype !== undefined && { subtype: existingAny.subtype }), + ...(((params.visibility ?? existingAny.visibility) ?? 'public') !== 'public' && { + visibility: params.visibility ?? existingAny.visibility + }), weight: updatedMetadata.weight, metadata: newMetadata, data: updatedMetadata.data, @@ -2335,6 +2489,14 @@ export class Brainy implements BrainyInterface { filter.service = params.service } + // Visibility (8.0): exclude hidden tiers by default. The exclusion is applied in the + // storage scan AFTER metadata load (where verb.visibility is known), so the storage + // limit stays exact. Like subtype, it disqualifies the metadata-less fast paths. + const excludedTiers = this.excludedVisibilityTiers(params) + if (excludedTiers) { + filter.excludeVisibility = excludedTiers + } + // VFS relationships are no longer filtered // VFS is part of the knowledge graph - users can filter explicitly if needed @@ -3004,6 +3166,13 @@ export class Brainy implements BrainyInterface { return this.findAggregate(params) } + // Visibility (8.0): resolve the ids to hide for this read ONCE. Default hides + // internal + system; opt back in with includeInternal / includeSystem. Applied as a + // hard candidate filter at every candidate-gathering point below so limit/offset stay + // correct. Empty set when both opts are set (nothing excluded). + const hiddenIds = await this.resolveHiddenIds(params) + const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id) + const startTime = Date.now() const result = await (async () => { let results: Result[] = [] @@ -3079,6 +3248,9 @@ export class Brainy implements BrainyInterface { filteredIds = await this.metadataIndex.getIdsForFilter(filter) } + // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact. + if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) + // Paginate BEFORE loading entities (production-scale!) const limit = params.limit || 10 const offset = params.offset || 0 @@ -3105,7 +3277,8 @@ export class Brainy implements BrainyInterface { // Unfiltered sort: column store handles this at O(K log S) scale. // No per-entity storage reads, no bucketing precision loss. if (params.orderBy) { - const k = limit + offset + // Over-fetch by the hidden count so dropping hidden ids still fills the page. + const k = limit + offset + hiddenIds.size const sortedIntIds = await this.metadataIndex.columnStore.sortTopK( params.orderBy, params.order || 'asc', @@ -3114,9 +3287,11 @@ export class Brainy implements BrainyInterface { // Convert int IDs to UUIDs and paginate const idMapper = this.metadataIndex.getIdMapper() - const allUuids = sortedIntIds + let allUuids = sortedIntIds .map(intId => idMapper.getUuid(intId)) .filter((uuid): uuid is string => uuid !== undefined) + // Visibility hard filter — drop hidden ids BEFORE pagination. + if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) const entitiesMap = await this.batchGet(pageIds) @@ -3137,9 +3312,13 @@ export class Brainy implements BrainyInterface { filter.isVFSEntity = { ne: true } } - // Use metadata index if we need to filter + // Use metadata index when there's an actual filter (e.g. excludeVFS). The empty + // filter returns nothing from getIdsForFilter, so the unfiltered case below uses + // getNouns instead (it returns all nouns, including their visibility). if (Object.keys(filter).length > 0) { - const filteredIds = await this.metadataIndex.getIdsForFilter(filter) + let filteredIds = await this.metadataIndex.getIdsForFilter(filter) + // Visibility hard filter — drop hidden ids BEFORE pagination. + if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance @@ -3151,13 +3330,19 @@ export class Brainy implements BrainyInterface { } } } else { - // No filtering needed, use direct storage query + // No metadata filter, use direct storage query. Over-fetch by the hidden count + // so the visibility filter below still fills the requested page (limit-exact). const storageResults = await this.storage.getNouns({ - pagination: { limit: limit + offset, offset: 0 } + pagination: { limit: limit + offset + hiddenIds.size, offset: 0 } }) - for (let i = offset; i < Math.min(offset + limit, storageResults.items.length); i++) { - const noun = storageResults.items[i] + // Visibility hard filter on the materialized nouns (each carries `visibility`). + const visible = hiddenIds.size > 0 + ? storageResults.items.filter((noun) => noun && !hiddenIds.has(noun.id)) + : storageResults.items + + for (let i = offset; i < Math.min(offset + limit, visible.length); i++) { + const noun = visible[i] if (noun) { const entity = await this.convertNounToEntity(noun) results.push(this.createResult(noun.id, 1.0, entity)) @@ -3212,6 +3397,11 @@ export class Brainy implements BrainyInterface { } preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter) + // Visibility hard filter — restrict the HNSW candidate set to non-hidden ids. + if (hiddenIds.size > 0) { + preResolvedMetadataIds = preResolvedMetadataIds.filter((id) => !hiddenIds.has(id)) + } + // Short-circuit: if metadata filter matches nothing, skip expensive vector search if (preResolvedMetadataIds.length === 0) { return [] @@ -3275,6 +3465,13 @@ export class Brainy implements BrainyInterface { results = Array.from(uniqueResults.values()) } + // Visibility hard filter for vector/text/proximity candidates (the pre-resolved + // path already excluded hidden ids; this covers searches with no metadata filter). + // Applied BEFORE the final sort+slice so limit/offset remain exact. + if (hiddenIds.size > 0 && results.length > 0) { + results = results.filter((r) => !isHidden(r.id)) + } + // Apply metadata filtering using pre-resolved IDs (metadata-first optimization). // When vector search was performed, HNSW already filtered by candidateIds — // this block handles pagination, entity loading, and metadata-only queries. @@ -8821,6 +9018,7 @@ export class Brainy implements BrainyInterface { to: v.targetId, type: (v.verb || v.type) as VerbType, ...(va.subtype !== undefined && { subtype: va.subtype as string }), + ...(v.visibility !== undefined && { visibility: v.visibility }), weight: v.weight ?? 1.0, data: v.data, metadata: v.metadata, diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 5d55ec1f..765f58e4 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -204,6 +204,38 @@ export interface VerbMetadata { * * Used for API responses and storage retrieval. */ + +/** + * Visibility tier for an entity or relationship — controls whether it surfaces + * on Brainy's default user-facing read paths. A reserved, top-level field; the + * absence of the field is exactly equivalent to `'public'`. + * + * - `'public'` (DEFAULT, and the meaning when the field is absent) — counted and + * returned everywhere: `find()`, `related()`, `getNounCount()`/`getVerbCount()`, + * `stats()`. + * - `'internal'` — a consumer's app-internal data. HIDDEN from the default + * `find()` / `related()` / counts / `stats()`, but retrievable with an explicit + * opt-in (`find({ includeInternal: true })`, `related({ includeInternal: true })`). + * - `'system'` — Brainy's own plumbing (e.g. the VFS root entity). Hidden + * EVERYWHERE by default and surfaced only via the explicit `includeSystem` + * opt-in. NOT settable by consumer code — only internal Brainy code assigns it, + * which is why the `add()` / `relate()` params type narrows to + * `'public' | 'internal'`. + */ +export type EntityVisibility = 'public' | 'internal' | 'system' + +/** + * Whether an entity/relationship of the given visibility is counted on the + * user-facing surfaces (`getNounCount()`/`getVerbCount()`, per-type stats). Only + * `'public'` (and absent, which means public) is counted; `'internal'` and + * `'system'` are not. + * @param visibility - The stored `visibility` value (may be absent/unknown). + * @returns `true` when the entity is counted on the user-facing surfaces. + */ +export function isCountedVisibility(visibility: unknown): boolean { + return visibility !== 'internal' && visibility !== 'system' +} + export interface HNSWNounWithMetadata { // HNSW Core (unchanged) id: string @@ -222,6 +254,14 @@ export interface HNSWNounWithMetadata { // fast path, never falling through to the metadata fallback. subtype?: string + // VISIBILITY — three-value tier gating whether this entity surfaces on default + // user-facing reads. Absent === 'public' (counted + returned everywhere). 'internal' + // hides the entity from default find()/counts/stats but keeps it retrievable via + // find({ includeInternal: true }). 'system' (Brainy plumbing) is hidden everywhere + // unless find({ includeSystem: true }). Stored only when not 'public' so the common + // case stays lean. See {@link EntityVisibility}. + visibility?: EntityVisibility + // QUALITY METRICS (top-level, explicit) confidence?: number weight?: number @@ -264,6 +304,7 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet = new Set([ 'level', 'type', 'subtype', + 'visibility', 'confidence', 'weight', 'createdAt', @@ -323,6 +364,7 @@ export const STANDARD_VERB_FIELDS: ReadonlySet = new Set([ 'sourceId', 'targetId', 'subtype', + 'visibility', 'confidence', 'weight', 'createdAt', @@ -394,6 +436,13 @@ export interface HNSWVerbWithMetadata { // the metadata fallback. subtype?: string + // VISIBILITY — verb mirror of the entity tier. Absent === 'public' (counted + + // returned everywhere). 'internal' hides the edge from default related()/counts/stats + // but keeps it retrievable via related({ includeInternal: true }). 'system' is hidden + // everywhere unless related({ includeSystem: true }). Stored only when not 'public'. + // See {@link EntityVisibility}. + visibility?: EntityVisibility + // QUALITY METRICS (top-level, explicit) weight?: number confidence?: number @@ -427,6 +476,7 @@ export interface GraphVerb { connections?: Map> // Optional connections from HNSW index type?: string // Optional type of the relationship subtype?: string // Optional sub-classification within the VerbType (7.30+) + visibility?: EntityVisibility // Visibility tier; absent === 'public'. See {@link EntityVisibility}. weight?: number // Optional weight of the relationship confidence?: number // Optional confidence score (0-1) metadata?: any // Optional metadata for the verb diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index ec511292..c7daf1e8 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -13,8 +13,10 @@ import { VerbMetadata, HNSWNounWithMetadata, HNSWVerbWithMetadata, - StatisticsData + StatisticsData, + isCountedVisibility } from '../coreTypes.js' +import type { EntityVisibility } from '../coreTypes.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' import { validateNounType, validateVerbType } from '../utils/typeValidation.js' import { @@ -184,6 +186,18 @@ function getVerbMetadataPath(id: string): string { return `entities/verbs/${shard}/${id}/metadata.json` } +/** + * Extract the entity id from an ID-first metadata path. Both noun and verb + * metadata live at `entities////metadata.json`, so the id is + * the second-to-last path segment. + * @param path - A metadata.json path produced by `getNounMetadataPath` / `getVerbMetadataPath`. + * @returns The id segment, or `undefined` if the path doesn't have the expected shape. + */ +function idFromMetadataPath(path: string): string | undefined { + const segments = path.split('/') + return segments[segments.length - 2] +} + /** * Base storage adapter that implements common functionality * This is an abstract class that should be extended by specific storage adapters @@ -278,6 +292,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { protected verbSubtypeByIdCache = new Map() // Total: 676 bytes (99.2% reduction vs Map-based tracking) + /** + * In-memory map from noun id → its non-public `visibility` tier ('internal' | + * 'system'). Parallel to `nounTypeByIdCache`. Lets `saveNoun_internal()` + * (which has no metadata access in its signature) skip the user-facing + * `nounCountsByType` increment for hidden entities, and lets + * `deleteNounMetadata()` skip the matching decrement — keeping the counts + * symmetric. Sparse by design: public entities (the common case) get NO + * entry, so the absence of an id means "public, counted". + */ + protected nounVisibilityByIdCache = new Map() + + /** + * In-memory map from verb id → its non-public `visibility` tier. Verb-side + * mirror of `nounVisibilityByIdCache`. Sparse — public edges get no entry. + */ + protected verbVisibilityByIdCache = new Map() + // Type caches REMOVED - ID-first paths eliminate need for type lookups! // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json // Type is just a field in the metadata, indexed by MetadataIndexManager for queries @@ -904,7 +935,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } // Combine into HNSWNounWithMetadata - Extract standard fields to top-level - const { noun, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata + const { noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata return { id: vector.id, @@ -914,6 +945,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Standard fields at top-level type: (noun as NounType) || NounType.Thing, subtype: subtype as string | undefined, + visibility: visibility as EntityVisibility | undefined, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -943,13 +975,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { for (const noun of nouns) { const metadata = await this.getNounMetadata(noun.id) if (metadata) { - const { noun: nounType, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata + const { noun: nounType, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata nounsWithMetadata.push({ ...noun, // Standard fields at top-level type: (nounType as NounType) || NounType.Thing, subtype: subtype as string | undefined, + visibility: visibility as EntityVisibility | undefined, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -1023,7 +1056,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { } // Combine into HNSWVerbWithMetadata - Extract standard fields to top-level - const { subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata + const { subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata return { id: verb.id, @@ -1034,6 +1067,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId: verb.targetId, // Standard fields at top-level subtype: subtype as string | undefined, + visibility: visibility as EntityVisibility | undefined, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -1098,7 +1132,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { const verb = this.deserializeVerb(vectorData) // Extract standard fields to top-level - const { subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadataData + const { subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadataData results.set(id, { id: verb.id, @@ -1109,6 +1143,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId: verb.targetId, // Standard fields at top-level subtype: subtype as string | undefined, + visibility: visibility as EntityVisibility | undefined, createdAt: (createdAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(), confidence: confidence as number | undefined, @@ -1489,6 +1524,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { ...deserialized, type: (metadata.noun || 'thing') as NounType, subtype: (metadata as any).subtype as string | undefined, + visibility: (metadata as any).visibility as EntityVisibility | undefined, confidence: metadata.confidence, weight: metadata.weight, createdAt: metadata.createdAt @@ -1569,6 +1605,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId?: string | string[] service?: string | string[] metadata?: Record + /** + * 8.0 visibility: tiers to exclude from results (e.g. `['internal','system']`). + * Applied after metadata load in the full scan, so it disqualifies the + * metadata-less graph-index fast paths (same as `subtype`). + */ + excludeVisibility?: string[] } }): Promise<{ items: HNSWVerbWithMetadata[] @@ -1599,6 +1641,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { : [(filter as any).subtype] ) : null + // 8.0 visibility exclusion — applied after metadata load (verb visibility lives in + // metadata), so hidden edges are skipped BEFORE the pagination window fills. + const excludeVisibility = (filter as { excludeVisibility?: string[] } | undefined)?.excludeVisibility + const filterExcludeVisibility = excludeVisibility && excludeVisibility.length > 0 + ? new Set(excludeVisibility) + : null // Iterate by shards (0x00-0xFF) instead of types - single pass! for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) { @@ -1645,10 +1693,20 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } + // Apply visibility exclusion (8.0). Absent === 'public' (kept); a stored + // 'internal'/'system' value is dropped when its tier is excluded. + if (filterExcludeVisibility) { + const visibility = (metadata as any)?.visibility as string | undefined + if (visibility && filterExcludeVisibility.has(visibility)) { + continue + } + } + // Combine verb + metadata collectedVerbs.push({ ...verb, subtype: (metadata as any)?.subtype as string | undefined, + visibility: (metadata as any)?.visibility as EntityVisibility | undefined, weight: metadata?.weight, confidence: metadata?.confidence, createdAt: metadata?.createdAt @@ -1701,6 +1759,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { targetId?: string | string[] service?: string | string[] metadata?: Record + /** + * 8.0 visibility: tiers to exclude from results (e.g. `['internal','system']`). + * Applied after metadata load in the full scan, so it disqualifies the + * metadata-less graph-index fast paths (same as `subtype`). + */ + excludeVisibility?: string[] } }): Promise<{ items: HNSWVerbWithMetadata[] @@ -1721,8 +1785,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { // fallthrough to the full shard scan (which loads metadata and applies the // subtype check after the load). Subtype is not stored on the raw HNSWVerb; // it's a metadata field. The graph-index fast paths return verbs without - // metadata, so they can't apply subtype filtering correctly. - if (options?.filter && !(options.filter as any).subtype) { + // metadata, so they can't apply subtype filtering correctly. The 8.0 + // `excludeVisibility` filter (also a metadata field) disqualifies them the same way. + if ( + options?.filter && + !(options.filter as any).subtype && + !(options.filter as { excludeVisibility?: string[] }).excludeVisibility + ) { // CRITICAL VFS FIX: If filtering by sourceId + verbType (most common VFS pattern!) // This is the query PathResolver.getChildren() uses: getRelations({ from: dirId, type: VerbType.Contains }) if ( @@ -2311,16 +2380,40 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.nounSubtypeByIdCache.delete(id) } + // Visibility (8.0): only public entities count toward the user-facing totals. + // Warm `nounVisibilityByIdCache` so `saveNoun_internal()` (no metadata access) + // can gate the `nounCountsByType` increment, and `deleteNounMetadata()` can gate + // the matching decrement. Sparse — public ids get no entry. + const newVisibility = metadata.visibility + const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) + const isCounted = isCountedVisibility(newVisibility) + if (isCounted) { + this.nounVisibilityByIdCache.delete(id) + } else { + this.nounVisibilityByIdCache.set(id, newVisibility as EntityVisibility) + } + // CRITICAL FIX: Increment count for new entities // This runs AFTER metadata is saved, guaranteeing type information is available // Uses synchronous increment since storage operations are already serialized // Fixes Bug #1: Count synchronization failure during add() and import() - if (isNew && metadata.noun) { + // 8.0: skip the user-facing total for internal/system entities (counts.json + getNounCount()). + if (isNew && metadata.noun && isCounted) { this.incrementEntityCount(metadata.noun) // Persist counts asynchronously (fire and forget) this.scheduleCountPersist().catch(() => { // Ignore persist errors - will retry on next operation }) + } else if (!isNew && metadata.noun && wasCounted !== isCounted) { + // Visibility flipped on update(): move the entity in/out of the user-facing total + // (counts.json / getNounCount()). `nounCountsByType` is gated independently in + // `saveNoun_internal()` off the cache warmed just above, so it is not touched here. + if (isCounted) { + this.incrementEntityCount(metadata.noun) + } else { + this.decrementEntityCount(metadata.noun) + } + this.scheduleCountPersist().catch(() => {}) } } @@ -2707,11 +2800,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { // decrement `nounCountsByType` so type-statistics stay honest across // deletes; symmetric with the increment in `saveNounMetadata_internal()`. const priorType = this.nounTypeByIdCache.get(id) + // 8.0 visibility: an internal/system entity was never added to `nounCountsByType` + // (gated in `saveNoun_internal()`), so it must not be decremented here either. + const priorCounted = isCountedVisibility(this.nounVisibilityByIdCache.get(id)) + this.nounVisibilityByIdCache.delete(id) if (priorType) { this.nounTypeByIdCache.delete(id) - const idx = TypeUtils.getNounIndex(priorType) - if (this.nounCountsByType[idx] > 0) { - this.nounCountsByType[idx]-- + if (priorCounted) { + const idx = TypeUtils.getNounIndex(priorType) + if (this.nounCountsByType[idx] > 0) { + this.nounCountsByType[idx]-- + } } // Symmetric subtype decrement @@ -2797,16 +2896,52 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.verbSubtypeByIdCache.delete(id) } + // Visibility (8.0): verb mirror of the noun count gating. Warm + // `verbVisibilityByIdCache` so `deleteVerbMetadata()` can prune it. Sparse — + // public edges get no entry. + // + // NOTE on `verbCountsByType`: unlike the noun path, `saveVerb_internal()` runs + // BEFORE this method (relate() saves the verb vector first) and has already done + // an UNCONDITIONAL `verbCountsByType[idx]++`. We therefore COMPENSATE here: for a + // new hidden edge, undo that bump. `updateRelation()` does not re-run + // `saveVerb_internal()`, so on a visibility flip we adjust the bucket directly. + const newVisibility = metadata.visibility + const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) + const isCounted = isCountedVisibility(newVisibility) + if (isCounted) { + this.verbVisibilityByIdCache.delete(id) + } else { + this.verbVisibilityByIdCache.set(id, newVisibility as EntityVisibility) + } + const verbTypeIdx = TypeUtils.getVerbIndex(verbType) + // CRITICAL FIX: Increment verb count for new relationships // This runs AFTER metadata is saved // Uses synchronous increment since storage operations are already serialized // Fixes Bug #2: Count synchronization failure during relate() and import() + // 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()). if (isNew) { - this.incrementVerbCount(verbType) + if (isCounted) { + this.incrementVerbCount(verbType) + } else { + // Hidden edge: undo the unconditional bump from saveVerb_internal(). + if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]-- + } // Persist counts asynchronously (fire and forget) this.scheduleCountPersist().catch(() => { // Ignore persist errors - will retry on next operation }) + } else if (wasCounted !== isCounted) { + // Visibility flipped on updateRelation() (saveVerb_internal did not run): move the + // edge in/out of both the user-facing total and the per-type bucket. + if (isCounted) { + this.incrementVerbCount(verbType) + this.verbCountsByType[verbTypeIdx]++ + } else { + this.decrementVerbCount(verbType) + if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]-- + } + this.scheduleCountPersist().catch(() => {}) } } @@ -2845,6 +2980,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.verbSubtypeByIdCache.delete(id) this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype) } + // 8.0 visibility: prune the cache entry (verb deletes don't touch verbCountsByType + // in this path, mirroring the existing verb-subtype delete semantics). + this.verbVisibilityByIdCache.delete(id) } // ============================================================================ @@ -3276,9 +3414,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { const metadata = await this.readWithInheritance(path) if (metadata && metadata.noun) { - const typeIndex = TypeUtils.getNounIndex(metadata.noun) - if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { - this.nounCountsByType[typeIndex]++ + // 8.0 visibility: rebuild only public entities into the user-facing + // per-type stat; warm the cache so later writes/deletes stay symmetric. + const id = idFromMetadataPath(path) + if (isCountedVisibility(metadata.visibility)) { + const typeIndex = TypeUtils.getNounIndex(metadata.noun) + if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { + this.nounCountsByType[typeIndex]++ + } + if (id) this.nounVisibilityByIdCache.delete(id) + } else if (id) { + this.nounVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) } } } catch (error) { @@ -3304,9 +3450,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { const metadata = await this.readWithInheritance(path) if (metadata && metadata.verb) { - const typeIndex = TypeUtils.getVerbIndex(metadata.verb) - if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { - this.verbCountsByType[typeIndex]++ + // 8.0 visibility: rebuild only public edges into the user-facing per-type stat. + const id = idFromMetadataPath(path) + if (isCountedVisibility(metadata.visibility)) { + const typeIndex = TypeUtils.getVerbIndex(metadata.verb) + if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { + this.verbCountsByType[typeIndex]++ + } + if (id) this.verbVisibilityByIdCache.delete(id) + } else if (id) { + this.verbVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) } } } catch (error) { @@ -3483,17 +3636,24 @@ export abstract class BaseStorage extends BaseStorageAdapter { const type = this.getNounType(noun) const path = getNounVectorPath(noun.id) - // Update type tracking + // Update type tracking — but only for entities that count toward the + // user-facing per-type stats. `nounVisibilityByIdCache` (warmed by + // `saveNounMetadata_internal()`) flags internal/system ids; public ids are + // absent, so the common case still increments. 8.0 visibility exclusion. const typeIndex = TypeUtils.getNounIndex(type) - this.nounCountsByType[typeIndex]++ + const counted = isCountedVisibility(this.nounVisibilityByIdCache.get(noun.id)) + if (counted) { + this.nounCountsByType[typeIndex]++ + } // COW-aware write: Use COW helper for branch isolation await this.writeObjectToBranch(path, noun) // Periodically save statistics // Also save on first noun of each type to ensure low-count types are tracked - const shouldSave = this.nounCountsByType[typeIndex] === 1 || // First noun of type - this.nounCountsByType[typeIndex] % 100 === 0 // Every 100th + const shouldSave = counted && + (this.nounCountsByType[typeIndex] === 1 || // First noun of type + this.nounCountsByType[typeIndex] % 100 === 0) // Every 100th if (shouldSave) { await this.saveTypeStatistics() } diff --git a/src/types/brainy.types.ts b/src/types/brainy.types.ts index 70055caa..9da3b32b 100644 --- a/src/types/brainy.types.ts +++ b/src/types/brainy.types.ts @@ -5,6 +5,7 @@ */ import { Vector } from '../coreTypes.js' +import type { EntityVisibility } from '../coreTypes.js' import { NounType, VerbType } from './graphTypes.js' // ============= Core Types ============= @@ -33,6 +34,13 @@ export interface Entity { * rolled into per-NounType statistics. */ subtype?: string + /** + * Visibility tier (reserved top-level field). Absent === `'public'` (counted and + * returned everywhere). `'internal'` hides the entity from default `find()` / counts / + * `stats()` (opt back in with `find({ includeInternal: true })`); `'system'` is Brainy + * plumbing, hidden unless `find({ includeSystem: true })`. See {@link EntityVisibility}. + */ + visibility?: EntityVisibility /** Opaque content — used for embeddings and semantic search. Not indexed by MetadataIndex. */ data?: any /** User-defined structured fields — indexed and queryable via `where` filters. */ @@ -84,6 +92,14 @@ export interface Relation { * the column-store directly. */ subtype?: string + /** + * Visibility tier (reserved top-level field), the verb mirror of `Entity.visibility`. + * Absent === `'public'` (counted and returned everywhere). `'internal'` hides the edge + * from default `related()` / counts / `stats()` (opt back in with + * `related({ includeInternal: true })`); `'system'` is Brainy plumbing. See + * {@link EntityVisibility}. + */ + visibility?: EntityVisibility /** Connection strength (0-1, default: 1.0) */ weight?: number /** Opaque content for the relationship (overrides auto-computed vector if provided) */ @@ -129,6 +145,7 @@ export interface Result { // Convenience: Common entity fields flattened to top level type?: NounType // Entity type (from entity.type) subtype?: string // Per-product sub-classification (from entity.subtype) + visibility?: EntityVisibility // Visibility tier (from entity.visibility); absent === 'public' metadata?: T // Entity metadata (from entity.metadata) data?: any // Entity data (from entity.data) confidence?: number // Type classification confidence (from entity.confidence) @@ -191,6 +208,15 @@ export interface AddParams { * aggregation on the standard-field fast path. */ subtype?: string + /** + * Visibility tier (reserved top-level field). Omit (or pass `'public'`) for normal + * data that is counted and returned everywhere. Pass `'internal'` for app-internal + * data that should be hidden from default `find()` / counts / `stats()` but stay + * retrievable via `find({ includeInternal: true })`. The `'system'` tier is reserved + * for Brainy's own plumbing and is intentionally not accepted here. Stored only when + * not `'public'`. + */ + visibility?: 'public' | 'internal' /** Structured queryable fields — indexed by MetadataIndex, used in `where` filters */ metadata?: T /** Custom entity ID (auto-generated UUID v4 if not provided) */ @@ -222,6 +248,13 @@ export interface UpdateParams { data?: any // New content to re-embed type?: NounType // Change type subtype?: string // Change subtype (set to '' or null-equivalent via dedicated unset is future work) + /** + * Change the visibility tier. Omit to preserve the existing value. Toggling between + * `'public'` and `'internal'` moves the entity in/out of the default-visible counts and + * `find()` results. The `'system'` tier is Brainy-internal (e.g. re-asserted on the VFS + * root during repair) — consumer code should only ever use `'public'` / `'internal'`. + */ + visibility?: EntityVisibility metadata?: Partial // Metadata to update merge?: boolean // Merge or replace metadata (default: true) vector?: Vector // New pre-computed vector @@ -259,6 +292,15 @@ export interface RelateParams { * (`getRelations({ verb, subtype })`) and aggregation (`groupBy:['subtype']`). */ subtype?: string + /** + * Visibility tier (reserved top-level field), the verb mirror of `AddParams.visibility`. + * Omit (or pass `'public'`) for normal edges that are counted and returned everywhere. + * Pass `'internal'` for app-internal edges hidden from default `related()` / counts / + * `stats()` but retrievable via `related({ includeInternal: true })`. The `'system'` + * tier is reserved for Brainy's own plumbing and is not accepted here. Stored only when + * not `'public'`. + */ + visibility?: 'public' | 'internal' /** Connection strength (0-1, default: 1.0) */ weight?: number /** Content for the relationship (optional — overrides auto-computed vector) */ @@ -282,6 +324,12 @@ export interface UpdateRelationParams { id: string // Relation to update type?: VerbType // Change verb type subtype?: string // Change sub-classification (omit to preserve existing) + /** + * Change the visibility tier. Omit to preserve the existing value. The verb mirror of + * `UpdateParams.visibility`; `'system'` is Brainy-internal — consumer code should only + * use `'public'` / `'internal'`. + */ + visibility?: EntityVisibility weight?: number // New weight confidence?: number // New confidence (0-1) data?: any // New content @@ -320,7 +368,24 @@ export interface FindParams { subtype?: string | string[] /** Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`) */ where?: Partial - + + // Visibility + /** + * Also return `visibility: 'internal'` entities. By default `find()` returns ONLY + * public entities (those with no `visibility` field, or `visibility: 'public'`); + * app-internal entities are hidden. Set `true` to include them. Applied as a hard + * candidate filter, so `limit` / `offset` stay correct. Does NOT include `'system'` + * entities — use `includeSystem` for those. + */ + includeInternal?: boolean + /** + * Also return `visibility: 'system'` entities (Brainy's own plumbing, e.g. the VFS + * root). Hidden by default and even when `includeInternal` is set. Set `true` only + * when you specifically need to see system entities. Applied as a hard candidate + * filter, so `limit` / `offset` stay correct. + */ + includeSystem?: boolean + // Graph Intelligence connected?: GraphConstraints @@ -481,6 +546,25 @@ export interface GetRelationsParams { */ subtype?: string | string[] + /** + * Also return `visibility: 'internal'` relationships. + * + * By default `related()` returns ONLY public relationships (those with no + * `visibility` field, or `visibility: 'public'`); app-internal edges are hidden. + * Set `true` to include them. Applied as a hard candidate filter so `limit` / + * `offset` stay correct. Does NOT include `'system'` edges — use `includeSystem`. + */ + includeInternal?: boolean + + /** + * Also return `visibility: 'system'` relationships (Brainy's own plumbing). + * + * Hidden by default and even when `includeInternal` is set. Set `true` only when + * you specifically need system edges. Applied as a hard candidate filter so + * `limit` / `offset` stay correct. + */ + includeSystem?: boolean + /** * Maximum number of results to return * diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 86b06ea5..771f2293 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -534,6 +534,7 @@ export function validateUpdateParams(params: UpdateParams): void { !params.type && !params.vector && params.subtype === undefined && + params.visibility === undefined && params.confidence === undefined && params.weight === undefined ) { diff --git a/src/utils/rebuildCounts.ts b/src/utils/rebuildCounts.ts index 389e7237..ce821772 100644 --- a/src/utils/rebuildCounts.ts +++ b/src/utils/rebuildCounts.ts @@ -75,6 +75,11 @@ export async function rebuildCounts(storage: BaseStorage): Promise