soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: visibility tier (public/internal/system) on nouns + verbs

Browse source 
Adds a reserved, top-level `visibility` field (mirrors the subtype rollout):
'public' (default, surfaced) | 'internal' (a consumer's app-internal data — hidden
from default find()/getRelations()/counts/stats, opt-in via includeInternal) |
'system' (Brainy plumbing, library-set only; the add()/relate() param narrows to
'public' | 'internal').

Fixes a real leak: the VFS root entity counted in getNounCount() and appeared in
find() (a fresh brain reported 1 entity). It is now visibility:'system' → excluded
from every user-facing surface (fresh brain reports 0).

- Reserved (STANDARD_ENTITY_FIELDS / STANDARD_VERB_FIELDS) — surfaced top-level on
  reads, never in the custom metadata bag; a 'public'/'internal' value smuggled
  through metadata is lifted to the field, 'system' dropped with a one-shot warning.
- Threaded through add/relate/update/updateRelation + their internal mirrors;
  stored only when not 'public' so the common case stays lean.
- Default exclusion in counts (baseStorage, isCountedVisibility), find()/getRelations()
  (hard candidate filter via excludeVisibility — keeps top-K/limit/offset correct),
  and rebuildCounts; includeInternal/includeSystem opt-ins.
- VFS root marked 'system'.

Same on-disk shape as the 8.0 line, so it carries forward unchanged. Backward-compatible:
absent === 'public', so all existing data stays counted + returned.

Tests: tests/unit/brainy/visibility.test.ts 17/17 (fresh-brain getNounCount()===0,
internal hidden + opt-in, verb symmetry, top-level surfacing, metadata-spoof rejection).
1522 unit green; count-synchronization integration green.
This commit is contained in:
David Snelling 2026-06-19 16:40:35 -07:00
parent c53dd61f96
commit 3a62445465
8 changed files with 778 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

224
src/brainy.ts
View file

@ -12,6 +12,7 @@ import { HNSWIndex } from './hnsw/hnswIndex.js'
import { createStorage } from './storage/storageFactory.js' import { createStorage } from './storage/storageFactory.js'
import { BaseStorage } from './storage/baseStorage.js' import { BaseStorage } from './storage/baseStorage.js'
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
import type { EntityVisibility } from './coreTypes.js'
import { import {
defaultEmbeddingFunction, defaultEmbeddingFunction,
cosineDistance, cosineDistance,
@ -1038,6 +1039,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Zero-config validation (static import for performance) // Zero-config validation (static import for performance)
validateAddParams(params) validateAddParams(params)
// Reserved 'visibility' arriving via the metadata bag is normalized to the
// top-level param, mirroring add()'s lift behavior for the other reserved
// fields. A 'public'/'internal' value is user-settable and remaps to
// params.visibility (top-level wins when both are supplied); 'system' is
// Brainy-only and is dropped with a one-shot warning. In every case the key
// is stripped from metadata so it never echoes inside the custom bag.
params = this.remapReservedVisibilityOnAdd('add', params)
// Tracked-field vocabulary enforcement (Layer 2). Walks both bags so a // Tracked-field vocabulary enforcement (Layer 2). Walks both bags so a
// tracked field declared at top level (e.g. 'subtype') and one declared in // tracked field declared at top level (e.g. 'subtype') and one declared in
// metadata (e.g. 'status') both validate. // metadata (e.g. 'status') both validate.
@ -1084,6 +1093,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
data: params.data, data: params.data,
noun: params.type, noun: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.subtype !== undefined && { subtype: params.subtype }),
// visibility: stored only when not 'public' (absent === public, keeps records lean)
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
service: params.service, service: params.service,
createdAt: Date.now(), createdAt: Date.now(),
updatedAt: Date.now(), updatedAt: Date.now(),
@ -1105,6 +1117,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
level: 0, level: 0,
type: params.type, type: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.weight !== undefined && { weight: params.weight }), ...(params.weight !== undefined && { weight: params.weight }),
createdAt: Date.now(), createdAt: Date.now(),
@ -1378,6 +1392,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Flatten common entity fields to top level // Flatten common entity fields to top level
type: entity.type, type: entity.type,
subtype: entity.subtype, subtype: entity.subtype,
visibility: entity.visibility,
metadata: entity.metadata, metadata: entity.metadata,
data: entity.data, data: entity.data,
confidence: entity.confidence, confidence: entity.confidence,
@ -1408,6 +1423,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
vector: noun.vector, vector: noun.vector,
type: noun.type || NounType.Thing, type: noun.type || NounType.Thing,
subtype: noun.subtype, subtype: noun.subtype,
visibility: noun.visibility,
// Standard fields at top-level // Standard fields at top-level
confidence: noun.confidence, confidence: noun.confidence,
@ -1451,13 +1467,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Extract standard fields, rest are custom metadata // Extract standard fields, rest are custom metadata
// Same destructuring as baseStorage.getNoun() to ensure consistency // Same destructuring as baseStorage.getNoun() to ensure consistency
const { noun, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata const { noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata
const entity: Entity<T> = { const entity: Entity<T> = {
id, id,
vector: [], // Stub vector (empty array - vectors not loaded for metadata-only) vector: [], // Stub vector (empty array - vectors not loaded for metadata-only)
type: noun as NounType || NounType.Thing, type: noun as NounType || NounType.Thing,
subtype, subtype,
visibility: visibility as EntityVisibility | undefined,
// Standard fields from metadata // Standard fields from metadata
confidence, confidence,
@ -1544,12 +1561,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (!params.metadata || typeof params.metadata !== 'object') return params if (!params.metadata || typeof params.metadata !== 'object') return params
const { const {
noun, subtype, createdAt, updatedAt, confidence, weight, service, noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service,
data, createdBy, _rev, ...customMetadata data, createdBy, _rev, ...customMetadata
} = params.metadata as Record<string, unknown> } = params.metadata as Record<string, unknown>
const hasReserved = const hasReserved =
noun !== undefined || subtype !== undefined || createdAt !== undefined || noun !== undefined || subtype !== undefined || visibility !== undefined ||
createdAt !== undefined ||
updatedAt !== undefined || confidence !== undefined || weight !== undefined || updatedAt !== undefined || confidence !== undefined || weight !== undefined ||
service !== undefined || data !== undefined || createdBy !== undefined || service !== undefined || data !== undefined || createdBy !== undefined ||
_rev !== undefined _rev !== undefined
@ -1572,16 +1590,125 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (service !== undefined) warnDropped('service', 'nothing — fixed at add() time') if (service !== undefined) warnDropped('service', 'nothing — fixed at add() time')
if (createdBy !== undefined) warnDropped('createdBy', 'nothing — fixed at add() time') if (createdBy !== undefined) warnDropped('createdBy', 'nothing — fixed at add() time')
if (_rev !== undefined) warnDropped('_rev', "the 'ifRev' param for optimistic concurrency") if (_rev !== undefined) warnDropped('_rev', "the 'ifRev' param for optimistic concurrency")
// 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal'
// value IS user-settable and is silently remapped to the top-level param below.)
if (visibility === 'system') {
warnDropped('visibility', "the 'visibility' param ('public' | 'internal')")
}
return { return {
...params, ...params,
metadata: customMetadata as UpdateParams<T>['metadata'], metadata: customMetadata as UpdateParams<T>['metadata'],
...(params.confidence === undefined && typeof confidence === 'number' && { confidence }), ...(params.confidence === undefined && typeof confidence === 'number' && { confidence }),
...(params.weight === undefined && typeof weight === 'number' && { weight }), ...(params.weight === undefined && typeof weight === 'number' && { weight }),
...(params.subtype === undefined && typeof subtype === 'string' && { subtype }) ...(params.subtype === undefined && typeof subtype === 'string' && { subtype }),
...(params.visibility === undefined &&
(visibility === 'public' || visibility === 'internal') && {
visibility: visibility as 'public' | 'internal'
})
} }
} }
/**
* @description Normalize a reserved `visibility` field that arrived inside the
* `metadata` bag of an `add()` / `relate()` call. Untyped (JavaScript) callers
* can smuggle the reserved key past the compile-time guard; this applies the
* same contract as the rest of the reserved fields:
* - `'public'` / `'internal'` — user-settable; remapped to the top-level
* `visibility` param unless the caller also passed it explicitly (top-level
* wins), and stripped from the metadata bag.
* - `'system'` — Brainy-only; dropped with a one-shot warning and stripped from
* the metadata bag (the entity/edge stays public).
* In every case the key is removed from `metadata` so it never echoes inside the
* custom bag.
* @param method - The calling method name (`'add'` | `'relate'`) for the warning text.
* @param params - The caller's params (not mutated; a normalized copy is returned).
* @returns Params with `visibility` normalized out of `metadata`.
*/
private remapReservedVisibilityOnAdd<
P extends { visibility?: 'public' | 'internal'; metadata?: any }
>(method: 'add' | 'relate', params: P): P {
if (!params.metadata || typeof params.metadata !== 'object') return params
const bag = params.metadata as Record<string, unknown>
if (!('visibility' in bag)) return params
const { visibility, ...customMetadata } = bag
if (visibility === 'system') {
this.warnDroppedReservedField(
method,
'visibility',
"the 'visibility' param ('public' | 'internal')"
)
}
return {
...params,
metadata: customMetadata,
...(params.visibility === undefined &&
(visibility === 'public' || visibility === 'internal') && {
visibility: visibility as 'public' | 'internal'
})
}
}
/**
* @description Emit a one-shot (per field, per process) warning that a reserved
* field supplied through a metadata bag was ignored, naming the correct write
* path. Shared by the add/relate/update reserved-field normalizers.
* @param method - The calling method name, used in the message.
* @param field - The reserved field that was dropped.
* @param rightPath - Human-readable description of where the value should go instead.
*/
private warnDroppedReservedField(method: string, field: string, rightPath: string): void {
if (Brainy.warnedReservedFields.has(field)) return
Brainy.warnedReservedFields.add(field)
prodLog.warn(
`[brainy] ${method}(): '${field}' is a reserved field and cannot be set ` +
`through a metadata patch — use ${rightPath}. The value was ignored. ` +
`(This warning is shown once per field per process.)`
)
}
/**
* The visibility tiers a read should EXCLUDE, given the caller's opt-ins.
* Default (no opts) hides both `'internal'` and `'system'`; `includeInternal`
* unhides internal; `includeSystem` unhides system. Returns `null` when nothing
* is excluded (both opts set) so callers can skip the filter entirely.
*
* @param params - The read params carrying `includeInternal` / `includeSystem`.
* @returns The set of excluded tier strings, or `null` for "exclude nothing".
*/
private excludedVisibilityTiers(
params: { includeInternal?: boolean; includeSystem?: boolean }
): EntityVisibility[] | null {
const excluded: EntityVisibility[] = []
if (!params.includeInternal) excluded.push('internal')
if (!params.includeSystem) excluded.push('system')
return excluded.length > 0 ? excluded : null
}
/**
* Resolve the set of entity/relationship ids to hide for this read, by asking
* the metadata index for everything tagged with an excluded visibility tier.
* Public entities carry NO stored `visibility` (absent === public), so they are
* never in this set — exactly the entities we want to keep. Returns an empty set
* when nothing is excluded. The result is used as a HARD candidate filter
* (subtracted from candidate ids BEFORE pagination), so `limit`/`offset` stay correct.
*
* @param params - The read params carrying the visibility opt-ins.
* @returns A set of ids to omit from results.
*/
private async resolveHiddenIds(
params: { includeInternal?: boolean; includeSystem?: boolean }
): Promise<Set<string>> {
const excluded = this.excludedVisibilityTiers(params)
if (!excluded) return new Set()
const ids = await this.metadataIndex.getIdsForFilter({
visibility: excluded.length === 1 ? excluded[0] : { oneOf: excluded }
})
return new Set(ids)
}
/** One-shot registry for reserved-field warnings (per process). */ /** One-shot registry for reserved-field warnings (per process). */
private static warnedReservedFields = new Set<string>() private static warnedReservedFields = new Set<string>()
@ -1672,7 +1799,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }),
// Update subtype if provided, otherwise preserve existing // Update subtype if provided, otherwise preserve existing
...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }) ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }),
// Visibility: take the new value if provided, else preserve existing. Stored only
// when the effective value is not 'public' (absent === public, keeps records lean).
// A change to 'public' therefore drops the field entirely.
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existing.visibility
})
} }
// Build entity structure for metadata index (with top-level fields) // Build entity structure for metadata index (with top-level fields)
@ -1683,6 +1816,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
level: 0, level: 0,
type: params.type || existing.type, type: params.type || existing.type,
subtype: params.subtype !== undefined ? params.subtype : existing.subtype, subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existing.visibility
}),
confidence: params.confidence !== undefined ? params.confidence : existing.confidence, confidence: params.confidence !== undefined ? params.confidence : existing.confidence,
weight: params.weight !== undefined ? params.weight : existing.weight, weight: params.weight !== undefined ? params.weight : existing.weight,
createdAt: existing.createdAt, createdAt: existing.createdAt,
@ -1976,6 +2112,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Zero-config validation (static import for performance) // Zero-config validation (static import for performance)
validateRelateParams(params) validateRelateParams(params)
// Reserved 'visibility' arriving via the metadata bag is normalized to the
// top-level param (mirror of add()): 'public'/'internal' lifts, 'system' is
// dropped with a one-shot warning, and the key is stripped from the bag.
params = this.remapReservedVisibilityOnAdd('relate', params)
// Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered
// via brain.requireSubtype() compose with the brain-wide strict-mode flag. // via brain.requireSubtype() compose with the brain-wide strict-mode flag.
// Metadata is passed so infrastructure edges (VFS containment) can bypass // Metadata is passed so infrastructure edges (VFS containment) can bypass
@ -2028,6 +2169,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.metadata || {}), ...(params.metadata || {}),
verb: params.type, verb: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.subtype !== undefined && { subtype: params.subtype }),
// visibility: stored only when not 'public' (absent === public, keeps records lean)
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
weight: params.weight ?? 1.0, weight: params.weight ?? 1.0,
createdAt: Date.now(), createdAt: Date.now(),
...((params as any).data !== undefined && { data: (params as any).data }) ...((params as any).data !== undefined && { data: (params as any).data })
@ -2044,6 +2188,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
verb: params.type, verb: params.type,
type: params.type, type: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
weight: params.weight ?? 1.0, weight: params.weight ?? 1.0,
metadata: params.metadata, metadata: params.metadata,
data: (params as any).data, data: (params as any).data,
@ -2213,6 +2359,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.subtype !== undefined ...(params.subtype !== undefined
? { subtype: params.subtype } ? { subtype: params.subtype }
: existingAny.subtype !== undefined && { subtype: existingAny.subtype }), : existingAny.subtype !== undefined && { subtype: existingAny.subtype }),
// Visibility: new value if provided, else preserve existing; stored only when the
// effective value is not 'public' (a change to 'public' drops the field).
...(((params.visibility ?? existingAny.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existingAny.visibility
}),
weight: params.weight ?? existingAny.weight ?? 1.0, weight: params.weight ?? existingAny.weight ?? 1.0,
...(params.confidence !== undefined ...(params.confidence !== undefined
? { confidence: params.confidence } ? { confidence: params.confidence }
@ -2237,6 +2388,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.subtype !== undefined ...(params.subtype !== undefined
? { subtype: params.subtype } ? { subtype: params.subtype }
: existingAny.subtype !== undefined && { subtype: existingAny.subtype }), : existingAny.subtype !== undefined && { subtype: existingAny.subtype }),
...(((params.visibility ?? existingAny.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existingAny.visibility
}),
weight: updatedMetadata.weight, weight: updatedMetadata.weight,
metadata: newMetadata, metadata: newMetadata,
data: updatedMetadata.data, data: updatedMetadata.data,
@ -2335,6 +2489,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
filter.service = params.service filter.service = params.service
} }
// Visibility (8.0): exclude hidden tiers by default. The exclusion is applied in the
// storage scan AFTER metadata load (where verb.visibility is known), so the storage
// limit stays exact. Like subtype, it disqualifies the metadata-less fast paths.
const excludedTiers = this.excludedVisibilityTiers(params)
if (excludedTiers) {
filter.excludeVisibility = excludedTiers
}
// VFS relationships are no longer filtered // VFS relationships are no longer filtered
// VFS is part of the knowledge graph - users can filter explicitly if needed // VFS is part of the knowledge graph - users can filter explicitly if needed
@ -3004,6 +3166,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return this.findAggregate(params) return this.findAggregate(params)
} }
// Visibility (8.0): resolve the ids to hide for this read ONCE. Default hides
// internal + system; opt back in with includeInternal / includeSystem. Applied as a
// hard candidate filter at every candidate-gathering point below so limit/offset stay
// correct. Empty set when both opts are set (nothing excluded).
const hiddenIds = await this.resolveHiddenIds(params)
const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id)
const startTime = Date.now() const startTime = Date.now()
const result = await (async () => { const result = await (async () => {
let results: Result<T>[] = [] let results: Result<T>[] = []
@ -3079,6 +3248,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
filteredIds = await this.metadataIndex.getIdsForFilter(filter) filteredIds = await this.metadataIndex.getIdsForFilter(filter)
} }
// Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact.
if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id))
// Paginate BEFORE loading entities (production-scale!) // Paginate BEFORE loading entities (production-scale!)
const limit = params.limit || 10 const limit = params.limit || 10
const offset = params.offset || 0 const offset = params.offset || 0
@ -3105,7 +3277,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Unfiltered sort: column store handles this at O(K log S) scale. // Unfiltered sort: column store handles this at O(K log S) scale.
// No per-entity storage reads, no bucketing precision loss. // No per-entity storage reads, no bucketing precision loss.
if (params.orderBy) { if (params.orderBy) {
const k = limit + offset // Over-fetch by the hidden count so dropping hidden ids still fills the page.
const k = limit + offset + hiddenIds.size
const sortedIntIds = await this.metadataIndex.columnStore.sortTopK( const sortedIntIds = await this.metadataIndex.columnStore.sortTopK(
params.orderBy, params.orderBy,
params.order || 'asc', params.order || 'asc',
@ -3114,9 +3287,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Convert int IDs to UUIDs and paginate // Convert int IDs to UUIDs and paginate
const idMapper = this.metadataIndex.getIdMapper() const idMapper = this.metadataIndex.getIdMapper()
const allUuids = sortedIntIds let allUuids = sortedIntIds
.map(intId => idMapper.getUuid(intId)) .map(intId => idMapper.getUuid(intId))
.filter((uuid): uuid is string => uuid !== undefined) .filter((uuid): uuid is string => uuid !== undefined)
// Visibility hard filter — drop hidden ids BEFORE pagination.
if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id))
const pageIds = allUuids.slice(offset, offset + limit) const pageIds = allUuids.slice(offset, offset + limit)
const entitiesMap = await this.batchGet(pageIds) const entitiesMap = await this.batchGet(pageIds)
@ -3137,9 +3312,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
filter.isVFSEntity = { ne: true } filter.isVFSEntity = { ne: true }
} }
// Use metadata index if we need to filter // Use metadata index when there's an actual filter (e.g. excludeVFS). The empty
// filter returns nothing from getIdsForFilter, so the unfiltered case below uses
// getNouns instead (it returns all nouns, including their visibility).
if (Object.keys(filter).length > 0) { if (Object.keys(filter).length > 0) {
const filteredIds = await this.metadataIndex.getIdsForFilter(filter) let filteredIds = await this.metadataIndex.getIdsForFilter(filter)
// Visibility hard filter — drop hidden ids BEFORE pagination.
if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id))
const pageIds = filteredIds.slice(offset, offset + limit) const pageIds = filteredIds.slice(offset, offset + limit)
// Batch-load entities for 10x faster cloud storage performance // Batch-load entities for 10x faster cloud storage performance
@ -3151,13 +3330,19 @@ export class Brainy<T = any> implements BrainyInterface<T> {
} }
} }
} else { } else {
// No filtering needed, use direct storage query // No metadata filter, use direct storage query. Over-fetch by the hidden count
// so the visibility filter below still fills the requested page (limit-exact).
const storageResults = await this.storage.getNouns({ const storageResults = await this.storage.getNouns({
pagination: { limit: limit + offset, offset: 0 } pagination: { limit: limit + offset + hiddenIds.size, offset: 0 }
}) })
for (let i = offset; i < Math.min(offset + limit, storageResults.items.length); i++) { // Visibility hard filter on the materialized nouns (each carries `visibility`).
const noun = storageResults.items[i] const visible = hiddenIds.size > 0
? storageResults.items.filter((noun) => noun && !hiddenIds.has(noun.id))
: storageResults.items
for (let i = offset; i < Math.min(offset + limit, visible.length); i++) {
const noun = visible[i]
if (noun) { if (noun) {
const entity = await this.convertNounToEntity(noun) const entity = await this.convertNounToEntity(noun)
results.push(this.createResult(noun.id, 1.0, entity)) results.push(this.createResult(noun.id, 1.0, entity))
@ -3212,6 +3397,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
} }
preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter) preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter)
// Visibility hard filter — restrict the HNSW candidate set to non-hidden ids.
if (hiddenIds.size > 0) {
preResolvedMetadataIds = preResolvedMetadataIds.filter((id) => !hiddenIds.has(id))
}
// Short-circuit: if metadata filter matches nothing, skip expensive vector search // Short-circuit: if metadata filter matches nothing, skip expensive vector search
if (preResolvedMetadataIds.length === 0) { if (preResolvedMetadataIds.length === 0) {
return [] return []
@ -3275,6 +3465,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
results = Array.from(uniqueResults.values()) results = Array.from(uniqueResults.values())
} }
// Visibility hard filter for vector/text/proximity candidates (the pre-resolved
// path already excluded hidden ids; this covers searches with no metadata filter).
// Applied BEFORE the final sort+slice so limit/offset remain exact.
if (hiddenIds.size > 0 && results.length > 0) {
results = results.filter((r) => !isHidden(r.id))
}
// Apply metadata filtering using pre-resolved IDs (metadata-first optimization). // Apply metadata filtering using pre-resolved IDs (metadata-first optimization).
// When vector search was performed, HNSW already filtered by candidateIds — // When vector search was performed, HNSW already filtered by candidateIds —
// this block handles pagination, entity loading, and metadata-only queries. // this block handles pagination, entity loading, and metadata-only queries.
@ -8821,6 +9018,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
to: v.targetId, to: v.targetId,
type: (v.verb || v.type) as VerbType, type: (v.verb || v.type) as VerbType,
...(va.subtype !== undefined && { subtype: va.subtype as string }), ...(va.subtype !== undefined && { subtype: va.subtype as string }),
...(v.visibility !== undefined && { visibility: v.visibility }),
weight: v.weight ?? 1.0, weight: v.weight ?? 1.0,
data: v.data, data: v.data,
metadata: v.metadata, metadata: v.metadata,

50
src/coreTypes.ts
View file

@ -204,6 +204,38 @@ export interface VerbMetadata {
* *
* Used for API responses and storage retrieval. * Used for API responses and storage retrieval.
*/ */
/**
* Visibility tier for an entity or relationship — controls whether it surfaces
* on Brainy's default user-facing read paths. A reserved, top-level field; the
* absence of the field is exactly equivalent to `'public'`.
*
* - `'public'` (DEFAULT, and the meaning when the field is absent) — counted and
* returned everywhere: `find()`, `related()`, `getNounCount()`/`getVerbCount()`,
* `stats()`.
* - `'internal'` — a consumer's app-internal data. HIDDEN from the default
* `find()` / `related()` / counts / `stats()`, but retrievable with an explicit
* opt-in (`find({ includeInternal: true })`, `related({ includeInternal: true })`).
* - `'system'` — Brainy's own plumbing (e.g. the VFS root entity). Hidden
* EVERYWHERE by default and surfaced only via the explicit `includeSystem`
* opt-in. NOT settable by consumer code — only internal Brainy code assigns it,
* which is why the `add()` / `relate()` params type narrows to
* `'public' | 'internal'`.
*/
export type EntityVisibility = 'public' | 'internal' | 'system'
/**
* Whether an entity/relationship of the given visibility is counted on the
* user-facing surfaces (`getNounCount()`/`getVerbCount()`, per-type stats). Only
* `'public'` (and absent, which means public) is counted; `'internal'` and
* `'system'` are not.
* @param visibility - The stored `visibility` value (may be absent/unknown).
* @returns `true` when the entity is counted on the user-facing surfaces.
*/
export function isCountedVisibility(visibility: unknown): boolean {
return visibility !== 'internal' && visibility !== 'system'
}
export interface HNSWNounWithMetadata { export interface HNSWNounWithMetadata {
// HNSW Core (unchanged) // HNSW Core (unchanged)
id: string id: string
@ -222,6 +254,14 @@ export interface HNSWNounWithMetadata {
// fast path, never falling through to the metadata fallback. // fast path, never falling through to the metadata fallback.
subtype?: string subtype?: string
// VISIBILITY — three-value tier gating whether this entity surfaces on default
// user-facing reads. Absent === 'public' (counted + returned everywhere). 'internal'
// hides the entity from default find()/counts/stats but keeps it retrievable via
// find({ includeInternal: true }). 'system' (Brainy plumbing) is hidden everywhere
// unless find({ includeSystem: true }). Stored only when not 'public' so the common
// case stays lean. See {@link EntityVisibility}.
visibility?: EntityVisibility
// QUALITY METRICS (top-level, explicit) // QUALITY METRICS (top-level, explicit)
confidence?: number confidence?: number
weight?: number weight?: number
@ -264,6 +304,7 @@ export const STANDARD_ENTITY_FIELDS: ReadonlySet<string> = new Set([
'level', 'level',
'type', 'type',
'subtype', 'subtype',
'visibility',
'confidence', 'confidence',
'weight', 'weight',
'createdAt', 'createdAt',
@ -323,6 +364,7 @@ export const STANDARD_VERB_FIELDS: ReadonlySet<string> = new Set([
'sourceId', 'sourceId',
'targetId', 'targetId',
'subtype', 'subtype',
'visibility',
'confidence', 'confidence',
'weight', 'weight',
'createdAt', 'createdAt',
@ -394,6 +436,13 @@ export interface HNSWVerbWithMetadata {
// the metadata fallback. // the metadata fallback.
subtype?: string subtype?: string
// VISIBILITY — verb mirror of the entity tier. Absent === 'public' (counted +
// returned everywhere). 'internal' hides the edge from default related()/counts/stats
// but keeps it retrievable via related({ includeInternal: true }). 'system' is hidden
// everywhere unless related({ includeSystem: true }). Stored only when not 'public'.
// See {@link EntityVisibility}.
visibility?: EntityVisibility
// QUALITY METRICS (top-level, explicit) // QUALITY METRICS (top-level, explicit)
weight?: number weight?: number
confidence?: number confidence?: number
@ -427,6 +476,7 @@ export interface GraphVerb {
connections?: Map<number, Set<string>> // Optional connections from HNSW index connections?: Map<number, Set<string>> // Optional connections from HNSW index
type?: string // Optional type of the relationship type?: string // Optional type of the relationship
subtype?: string // Optional sub-classification within the VerbType (7.30+) subtype?: string // Optional sub-classification within the VerbType (7.30+)
visibility?: EntityVisibility // Visibility tier; absent === 'public'. See {@link EntityVisibility}.
weight?: number // Optional weight of the relationship weight?: number // Optional weight of the relationship
confidence?: number // Optional confidence score (0-1) confidence?: number // Optional confidence score (0-1)
metadata?: any // Optional metadata for the verb metadata?: any // Optional metadata for the verb

204
src/storage/baseStorage.ts
View file

@ -13,8 +13,10 @@ import {
VerbMetadata, VerbMetadata,
HNSWNounWithMetadata, HNSWNounWithMetadata,
HNSWVerbWithMetadata, HNSWVerbWithMetadata,
StatisticsData StatisticsData,
isCountedVisibility
} from '../coreTypes.js' } from '../coreTypes.js'
import type { EntityVisibility } from '../coreTypes.js'
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
import { validateNounType, validateVerbType } from '../utils/typeValidation.js' import { validateNounType, validateVerbType } from '../utils/typeValidation.js'
import { import {
@ -184,6 +186,18 @@ function getVerbMetadataPath(id: string): string {
return `entities/verbs/${shard}/${id}/metadata.json` return `entities/verbs/${shard}/${id}/metadata.json`
} }
/**
* Extract the entity id from an ID-first metadata path. Both noun and verb
* metadata live at `entities/<kind>/<shard>/<id>/metadata.json`, so the id is
* the second-to-last path segment.
* @param path - A metadata.json path produced by `getNounMetadataPath` / `getVerbMetadataPath`.
* @returns The id segment, or `undefined` if the path doesn't have the expected shape.
*/
function idFromMetadataPath(path: string): string | undefined {
const segments = path.split('/')
return segments[segments.length - 2]
}
/** /**
* Base storage adapter that implements common functionality * Base storage adapter that implements common functionality
* This is an abstract class that should be extended by specific storage adapters * This is an abstract class that should be extended by specific storage adapters
@ -278,6 +292,23 @@ export abstract class BaseStorage extends BaseStorageAdapter {
protected verbSubtypeByIdCache = new Map<string, { verb: VerbType; subtype: string }>() protected verbSubtypeByIdCache = new Map<string, { verb: VerbType; subtype: string }>()
// Total: 676 bytes (99.2% reduction vs Map-based tracking) // Total: 676 bytes (99.2% reduction vs Map-based tracking)
/**
* In-memory map from noun id → its non-public `visibility` tier ('internal' |
* 'system'). Parallel to `nounTypeByIdCache`. Lets `saveNoun_internal()`
* (which has no metadata access in its signature) skip the user-facing
* `nounCountsByType` increment for hidden entities, and lets
* `deleteNounMetadata()` skip the matching decrement — keeping the counts
* symmetric. Sparse by design: public entities (the common case) get NO
* entry, so the absence of an id means "public, counted".
*/
protected nounVisibilityByIdCache = new Map<string, EntityVisibility>()
/**
* In-memory map from verb id → its non-public `visibility` tier. Verb-side
* mirror of `nounVisibilityByIdCache`. Sparse — public edges get no entry.
*/
protected verbVisibilityByIdCache = new Map<string, EntityVisibility>()
// Type caches REMOVED - ID-first paths eliminate need for type lookups! // Type caches REMOVED - ID-first paths eliminate need for type lookups!
// With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json
// Type is just a field in the metadata, indexed by MetadataIndexManager for queries // Type is just a field in the metadata, indexed by MetadataIndexManager for queries
@ -904,7 +935,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
} }
// Combine into HNSWNounWithMetadata - Extract standard fields to top-level // Combine into HNSWNounWithMetadata - Extract standard fields to top-level
const { noun, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata const { noun, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata
return { return {
id: vector.id, id: vector.id,
@ -914,6 +945,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// Standard fields at top-level // Standard fields at top-level
type: (noun as NounType) || NounType.Thing, type: (noun as NounType) || NounType.Thing,
subtype: subtype as string | undefined, subtype: subtype as string | undefined,
visibility: visibility as EntityVisibility | undefined,
createdAt: (createdAt as number) || Date.now(), createdAt: (createdAt as number) || Date.now(),
updatedAt: (updatedAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(),
confidence: confidence as number | undefined, confidence: confidence as number | undefined,
@ -943,13 +975,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
for (const noun of nouns) { for (const noun of nouns) {
const metadata = await this.getNounMetadata(noun.id) const metadata = await this.getNounMetadata(noun.id)
if (metadata) { if (metadata) {
const { noun: nounType, subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata const { noun: nounType, subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata
nounsWithMetadata.push({ nounsWithMetadata.push({
...noun, ...noun,
// Standard fields at top-level // Standard fields at top-level
type: (nounType as NounType) || NounType.Thing, type: (nounType as NounType) || NounType.Thing,
subtype: subtype as string | undefined, subtype: subtype as string | undefined,
visibility: visibility as EntityVisibility | undefined,
createdAt: (createdAt as number) || Date.now(), createdAt: (createdAt as number) || Date.now(),
updatedAt: (updatedAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(),
confidence: confidence as number | undefined, confidence: confidence as number | undefined,
@ -1023,7 +1056,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
} }
// Combine into HNSWVerbWithMetadata - Extract standard fields to top-level // Combine into HNSWVerbWithMetadata - Extract standard fields to top-level
const { subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata const { subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadata
return { return {
id: verb.id, id: verb.id,
@ -1034,6 +1067,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
targetId: verb.targetId, targetId: verb.targetId,
// Standard fields at top-level // Standard fields at top-level
subtype: subtype as string | undefined, subtype: subtype as string | undefined,
visibility: visibility as EntityVisibility | undefined,
createdAt: (createdAt as number) || Date.now(), createdAt: (createdAt as number) || Date.now(),
updatedAt: (updatedAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(),
confidence: confidence as number | undefined, confidence: confidence as number | undefined,
@ -1098,7 +1132,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const verb = this.deserializeVerb(vectorData) const verb = this.deserializeVerb(vectorData)
// Extract standard fields to top-level // Extract standard fields to top-level
const { subtype, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadataData const { subtype, visibility, createdAt, updatedAt, confidence, weight, service, data, createdBy, _rev, ...customMetadata } = metadataData
results.set(id, { results.set(id, {
id: verb.id, id: verb.id,
@ -1109,6 +1143,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
targetId: verb.targetId, targetId: verb.targetId,
// Standard fields at top-level // Standard fields at top-level
subtype: subtype as string | undefined, subtype: subtype as string | undefined,
visibility: visibility as EntityVisibility | undefined,
createdAt: (createdAt as number) || Date.now(), createdAt: (createdAt as number) || Date.now(),
updatedAt: (updatedAt as number) || Date.now(), updatedAt: (updatedAt as number) || Date.now(),
confidence: confidence as number | undefined, confidence: confidence as number | undefined,
@ -1489,6 +1524,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
...deserialized, ...deserialized,
type: (metadata.noun || 'thing') as NounType, type: (metadata.noun || 'thing') as NounType,
subtype: (metadata as any).subtype as string | undefined, subtype: (metadata as any).subtype as string | undefined,
visibility: (metadata as any).visibility as EntityVisibility | undefined,
confidence: metadata.confidence, confidence: metadata.confidence,
weight: metadata.weight, weight: metadata.weight,
createdAt: metadata.createdAt createdAt: metadata.createdAt
@ -1569,6 +1605,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
targetId?: string | string[] targetId?: string | string[]
service?: string | string[] service?: string | string[]
metadata?: Record<string, any> metadata?: Record<string, any>
/**
* 8.0 visibility: tiers to exclude from results (e.g. `['internal','system']`).
* Applied after metadata load in the full scan, so it disqualifies the
* metadata-less graph-index fast paths (same as `subtype`).
*/
excludeVisibility?: string[]
} }
}): Promise<{ }): Promise<{
items: HNSWVerbWithMetadata[] items: HNSWVerbWithMetadata[]
@ -1599,6 +1641,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
: [(filter as any).subtype] : [(filter as any).subtype]
) )
: null : null
// 8.0 visibility exclusion — applied after metadata load (verb visibility lives in
// metadata), so hidden edges are skipped BEFORE the pagination window fills.
const excludeVisibility = (filter as { excludeVisibility?: string[] } | undefined)?.excludeVisibility
const filterExcludeVisibility = excludeVisibility && excludeVisibility.length > 0
? new Set(excludeVisibility)
: null
// Iterate by shards (0x00-0xFF) instead of types - single pass! // Iterate by shards (0x00-0xFF) instead of types - single pass!
for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) { for (let shard = 0; shard < 256 && collectedVerbs.length < targetCount; shard++) {
@ -1645,10 +1693,20 @@ export abstract class BaseStorage extends BaseStorageAdapter {
} }
} }
// Apply visibility exclusion (8.0). Absent === 'public' (kept); a stored
// 'internal'/'system' value is dropped when its tier is excluded.
if (filterExcludeVisibility) {
const visibility = (metadata as any)?.visibility as string | undefined
if (visibility && filterExcludeVisibility.has(visibility)) {
continue
}
}
// Combine verb + metadata // Combine verb + metadata
collectedVerbs.push({ collectedVerbs.push({
...verb, ...verb,
subtype: (metadata as any)?.subtype as string | undefined, subtype: (metadata as any)?.subtype as string | undefined,
visibility: (metadata as any)?.visibility as EntityVisibility | undefined,
weight: metadata?.weight, weight: metadata?.weight,
confidence: metadata?.confidence, confidence: metadata?.confidence,
createdAt: metadata?.createdAt createdAt: metadata?.createdAt
@ -1701,6 +1759,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
targetId?: string | string[] targetId?: string | string[]
service?: string | string[] service?: string | string[]
metadata?: Record<string, any> metadata?: Record<string, any>
/**
* 8.0 visibility: tiers to exclude from results (e.g. `['internal','system']`).
* Applied after metadata load in the full scan, so it disqualifies the
* metadata-less graph-index fast paths (same as `subtype`).
*/
excludeVisibility?: string[]
} }
}): Promise<{ }): Promise<{
items: HNSWVerbWithMetadata[] items: HNSWVerbWithMetadata[]
@ -1721,8 +1785,13 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// fallthrough to the full shard scan (which loads metadata and applies the // fallthrough to the full shard scan (which loads metadata and applies the
// subtype check after the load). Subtype is not stored on the raw HNSWVerb; // subtype check after the load). Subtype is not stored on the raw HNSWVerb;
// it's a metadata field. The graph-index fast paths return verbs without // it's a metadata field. The graph-index fast paths return verbs without
// metadata, so they can't apply subtype filtering correctly. // metadata, so they can't apply subtype filtering correctly. The 8.0
if (options?.filter && !(options.filter as any).subtype) { // `excludeVisibility` filter (also a metadata field) disqualifies them the same way.
if (
options?.filter &&
!(options.filter as any).subtype &&
!(options.filter as { excludeVisibility?: string[] }).excludeVisibility
) {
// CRITICAL VFS FIX: If filtering by sourceId + verbType (most common VFS pattern!) // CRITICAL VFS FIX: If filtering by sourceId + verbType (most common VFS pattern!)
// This is the query PathResolver.getChildren() uses: getRelations({ from: dirId, type: VerbType.Contains }) // This is the query PathResolver.getChildren() uses: getRelations({ from: dirId, type: VerbType.Contains })
if ( if (
@ -2311,16 +2380,40 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.nounSubtypeByIdCache.delete(id) this.nounSubtypeByIdCache.delete(id)
} }
// Visibility (8.0): only public entities count toward the user-facing totals.
// Warm `nounVisibilityByIdCache` so `saveNoun_internal()` (no metadata access)
// can gate the `nounCountsByType` increment, and `deleteNounMetadata()` can gate
// the matching decrement. Sparse — public ids get no entry.
const newVisibility = metadata.visibility
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
const isCounted = isCountedVisibility(newVisibility)
if (isCounted) {
this.nounVisibilityByIdCache.delete(id)
} else {
this.nounVisibilityByIdCache.set(id, newVisibility as EntityVisibility)
}
// CRITICAL FIX: Increment count for new entities // CRITICAL FIX: Increment count for new entities
// This runs AFTER metadata is saved, guaranteeing type information is available // This runs AFTER metadata is saved, guaranteeing type information is available
// Uses synchronous increment since storage operations are already serialized // Uses synchronous increment since storage operations are already serialized
// Fixes Bug #1: Count synchronization failure during add() and import() // Fixes Bug #1: Count synchronization failure during add() and import()
if (isNew && metadata.noun) { // 8.0: skip the user-facing total for internal/system entities (counts.json + getNounCount()).
if (isNew && metadata.noun && isCounted) {
this.incrementEntityCount(metadata.noun) this.incrementEntityCount(metadata.noun)
// Persist counts asynchronously (fire and forget) // Persist counts asynchronously (fire and forget)
this.scheduleCountPersist().catch(() => { this.scheduleCountPersist().catch(() => {
// Ignore persist errors - will retry on next operation // Ignore persist errors - will retry on next operation
}) })
} else if (!isNew && metadata.noun && wasCounted !== isCounted) {
// Visibility flipped on update(): move the entity in/out of the user-facing total
// (counts.json / getNounCount()). `nounCountsByType` is gated independently in
// `saveNoun_internal()` off the cache warmed just above, so it is not touched here.
if (isCounted) {
this.incrementEntityCount(metadata.noun)
} else {
this.decrementEntityCount(metadata.noun)
}
this.scheduleCountPersist().catch(() => {})
} }
} }
@ -2707,11 +2800,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// decrement `nounCountsByType` so type-statistics stay honest across // decrement `nounCountsByType` so type-statistics stay honest across
// deletes; symmetric with the increment in `saveNounMetadata_internal()`. // deletes; symmetric with the increment in `saveNounMetadata_internal()`.
const priorType = this.nounTypeByIdCache.get(id) const priorType = this.nounTypeByIdCache.get(id)
// 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
// (gated in `saveNoun_internal()`), so it must not be decremented here either.
const priorCounted = isCountedVisibility(this.nounVisibilityByIdCache.get(id))
this.nounVisibilityByIdCache.delete(id)
if (priorType) { if (priorType) {
this.nounTypeByIdCache.delete(id) this.nounTypeByIdCache.delete(id)
const idx = TypeUtils.getNounIndex(priorType) if (priorCounted) {
if (this.nounCountsByType[idx] > 0) { const idx = TypeUtils.getNounIndex(priorType)
this.nounCountsByType[idx]-- if (this.nounCountsByType[idx] > 0) {
this.nounCountsByType[idx]--
}
} }
// Symmetric subtype decrement // Symmetric subtype decrement
@ -2797,16 +2896,52 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.verbSubtypeByIdCache.delete(id) this.verbSubtypeByIdCache.delete(id)
} }
// Visibility (8.0): verb mirror of the noun count gating. Warm
// `verbVisibilityByIdCache` so `deleteVerbMetadata()` can prune it. Sparse —
// public edges get no entry.
//
// NOTE on `verbCountsByType`: unlike the noun path, `saveVerb_internal()` runs
// BEFORE this method (relate() saves the verb vector first) and has already done
// an UNCONDITIONAL `verbCountsByType[idx]++`. We therefore COMPENSATE here: for a
// new hidden edge, undo that bump. `updateRelation()` does not re-run
// `saveVerb_internal()`, so on a visibility flip we adjust the bucket directly.
const newVisibility = metadata.visibility
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
const isCounted = isCountedVisibility(newVisibility)
if (isCounted) {
this.verbVisibilityByIdCache.delete(id)
} else {
this.verbVisibilityByIdCache.set(id, newVisibility as EntityVisibility)
}
const verbTypeIdx = TypeUtils.getVerbIndex(verbType)
// CRITICAL FIX: Increment verb count for new relationships // CRITICAL FIX: Increment verb count for new relationships
// This runs AFTER metadata is saved // This runs AFTER metadata is saved
// Uses synchronous increment since storage operations are already serialized // Uses synchronous increment since storage operations are already serialized
// Fixes Bug #2: Count synchronization failure during relate() and import() // Fixes Bug #2: Count synchronization failure during relate() and import()
// 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
if (isNew) { if (isNew) {
this.incrementVerbCount(verbType) if (isCounted) {
this.incrementVerbCount(verbType)
} else {
// Hidden edge: undo the unconditional bump from saveVerb_internal().
if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]--
}
// Persist counts asynchronously (fire and forget) // Persist counts asynchronously (fire and forget)
this.scheduleCountPersist().catch(() => { this.scheduleCountPersist().catch(() => {
// Ignore persist errors - will retry on next operation // Ignore persist errors - will retry on next operation
}) })
} else if (wasCounted !== isCounted) {
// Visibility flipped on updateRelation() (saveVerb_internal did not run): move the
// edge in/out of both the user-facing total and the per-type bucket.
if (isCounted) {
this.incrementVerbCount(verbType)
this.verbCountsByType[verbTypeIdx]++
} else {
this.decrementVerbCount(verbType)
if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]--
}
this.scheduleCountPersist().catch(() => {})
} }
} }
@ -2845,6 +2980,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.verbSubtypeByIdCache.delete(id) this.verbSubtypeByIdCache.delete(id)
this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype) this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype)
} }
// 8.0 visibility: prune the cache entry (verb deletes don't touch verbCountsByType
// in this path, mirroring the existing verb-subtype delete semantics).
this.verbVisibilityByIdCache.delete(id)
} }
// ============================================================================ // ============================================================================
@ -3276,9 +3414,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
try { try {
const metadata = await this.readWithInheritance(path) const metadata = await this.readWithInheritance(path)
if (metadata && metadata.noun) { if (metadata && metadata.noun) {
const typeIndex = TypeUtils.getNounIndex(metadata.noun) // 8.0 visibility: rebuild only public entities into the user-facing
if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { // per-type stat; warm the cache so later writes/deletes stay symmetric.
this.nounCountsByType[typeIndex]++ const id = idFromMetadataPath(path)
if (isCountedVisibility(metadata.visibility)) {
const typeIndex = TypeUtils.getNounIndex(metadata.noun)
if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) {
this.nounCountsByType[typeIndex]++
}
if (id) this.nounVisibilityByIdCache.delete(id)
} else if (id) {
this.nounVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility)
} }
} }
} catch (error) { } catch (error) {
@ -3304,9 +3450,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
try { try {
const metadata = await this.readWithInheritance(path) const metadata = await this.readWithInheritance(path)
if (metadata && metadata.verb) { if (metadata && metadata.verb) {
const typeIndex = TypeUtils.getVerbIndex(metadata.verb) // 8.0 visibility: rebuild only public edges into the user-facing per-type stat.
if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { const id = idFromMetadataPath(path)
this.verbCountsByType[typeIndex]++ if (isCountedVisibility(metadata.visibility)) {
const typeIndex = TypeUtils.getVerbIndex(metadata.verb)
if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) {
this.verbCountsByType[typeIndex]++
}
if (id) this.verbVisibilityByIdCache.delete(id)
} else if (id) {
this.verbVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility)
} }
} }
} catch (error) { } catch (error) {
@ -3483,17 +3636,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const type = this.getNounType(noun) const type = this.getNounType(noun)
const path = getNounVectorPath(noun.id) const path = getNounVectorPath(noun.id)
// Update type tracking // Update type tracking — but only for entities that count toward the
// user-facing per-type stats. `nounVisibilityByIdCache` (warmed by
// `saveNounMetadata_internal()`) flags internal/system ids; public ids are
// absent, so the common case still increments. 8.0 visibility exclusion.
const typeIndex = TypeUtils.getNounIndex(type) const typeIndex = TypeUtils.getNounIndex(type)
this.nounCountsByType[typeIndex]++ const counted = isCountedVisibility(this.nounVisibilityByIdCache.get(noun.id))
if (counted) {
this.nounCountsByType[typeIndex]++
}
// COW-aware write: Use COW helper for branch isolation // COW-aware write: Use COW helper for branch isolation
await this.writeObjectToBranch(path, noun) await this.writeObjectToBranch(path, noun)
// Periodically save statistics // Periodically save statistics
// Also save on first noun of each type to ensure low-count types are tracked // Also save on first noun of each type to ensure low-count types are tracked
const shouldSave = this.nounCountsByType[typeIndex] === 1 || // First noun of type const shouldSave = counted &&
this.nounCountsByType[typeIndex] % 100 === 0 // Every 100th (this.nounCountsByType[typeIndex] === 1 || // First noun of type
this.nounCountsByType[typeIndex] % 100 === 0) // Every 100th
if (shouldSave) { if (shouldSave) {
await this.saveTypeStatistics() await this.saveTypeStatistics()
} }

86
src/types/brainy.types.ts
View file

@ -5,6 +5,7 @@
*/ */
import { Vector } from '../coreTypes.js' import { Vector } from '../coreTypes.js'
import type { EntityVisibility } from '../coreTypes.js'
import { NounType, VerbType } from './graphTypes.js' import { NounType, VerbType } from './graphTypes.js'
// ============= Core Types ============= // ============= Core Types =============
@ -33,6 +34,13 @@ export interface Entity<T = any> {
* rolled into per-NounType statistics. * rolled into per-NounType statistics.
*/ */
subtype?: string subtype?: string
/**
* Visibility tier (reserved top-level field). Absent === `'public'` (counted and
* returned everywhere). `'internal'` hides the entity from default `find()` / counts /
* `stats()` (opt back in with `find({ includeInternal: true })`); `'system'` is Brainy
* plumbing, hidden unless `find({ includeSystem: true })`. See {@link EntityVisibility}.
*/
visibility?: EntityVisibility
/** Opaque content — used for embeddings and semantic search. Not indexed by MetadataIndex. */ /** Opaque content — used for embeddings and semantic search. Not indexed by MetadataIndex. */
data?: any data?: any
/** User-defined structured fields — indexed and queryable via `where` filters. */ /** User-defined structured fields — indexed and queryable via `where` filters. */
@ -84,6 +92,14 @@ export interface Relation<T = any> {
* the column-store directly. * the column-store directly.
*/ */
subtype?: string subtype?: string
/**
* Visibility tier (reserved top-level field), the verb mirror of `Entity.visibility`.
* Absent === `'public'` (counted and returned everywhere). `'internal'` hides the edge
* from default `related()` / counts / `stats()` (opt back in with
* `related({ includeInternal: true })`); `'system'` is Brainy plumbing. See
* {@link EntityVisibility}.
*/
visibility?: EntityVisibility
/** Connection strength (0-1, default: 1.0) */ /** Connection strength (0-1, default: 1.0) */
weight?: number weight?: number
/** Opaque content for the relationship (overrides auto-computed vector if provided) */ /** Opaque content for the relationship (overrides auto-computed vector if provided) */
@ -129,6 +145,7 @@ export interface Result<T = any> {
// Convenience: Common entity fields flattened to top level // Convenience: Common entity fields flattened to top level
type?: NounType // Entity type (from entity.type) type?: NounType // Entity type (from entity.type)
subtype?: string // Per-product sub-classification (from entity.subtype) subtype?: string // Per-product sub-classification (from entity.subtype)
visibility?: EntityVisibility // Visibility tier (from entity.visibility); absent === 'public'
metadata?: T // Entity metadata (from entity.metadata) metadata?: T // Entity metadata (from entity.metadata)
data?: any // Entity data (from entity.data) data?: any // Entity data (from entity.data)
confidence?: number // Type classification confidence (from entity.confidence) confidence?: number // Type classification confidence (from entity.confidence)
@ -191,6 +208,15 @@ export interface AddParams<T = any> {
* aggregation on the standard-field fast path. * aggregation on the standard-field fast path.
*/ */
subtype?: string subtype?: string
/**
* Visibility tier (reserved top-level field). Omit (or pass `'public'`) for normal
* data that is counted and returned everywhere. Pass `'internal'` for app-internal
* data that should be hidden from default `find()` / counts / `stats()` but stay
* retrievable via `find({ includeInternal: true })`. The `'system'` tier is reserved
* for Brainy's own plumbing and is intentionally not accepted here. Stored only when
* not `'public'`.
*/
visibility?: 'public' | 'internal'
/** Structured queryable fields — indexed by MetadataIndex, used in `where` filters */ /** Structured queryable fields — indexed by MetadataIndex, used in `where` filters */
metadata?: T metadata?: T
/** Custom entity ID (auto-generated UUID v4 if not provided) */ /** Custom entity ID (auto-generated UUID v4 if not provided) */
@ -222,6 +248,13 @@ export interface UpdateParams<T = any> {
data?: any // New content to re-embed data?: any // New content to re-embed
type?: NounType // Change type type?: NounType // Change type
subtype?: string // Change subtype (set to '' or null-equivalent via dedicated unset is future work) subtype?: string // Change subtype (set to '' or null-equivalent via dedicated unset is future work)
/**
* Change the visibility tier. Omit to preserve the existing value. Toggling between
* `'public'` and `'internal'` moves the entity in/out of the default-visible counts and
* `find()` results. The `'system'` tier is Brainy-internal (e.g. re-asserted on the VFS
* root during repair) — consumer code should only ever use `'public'` / `'internal'`.
*/
visibility?: EntityVisibility
metadata?: Partial<T> // Metadata to update metadata?: Partial<T> // Metadata to update
merge?: boolean // Merge or replace metadata (default: true) merge?: boolean // Merge or replace metadata (default: true)
vector?: Vector // New pre-computed vector vector?: Vector // New pre-computed vector
@ -259,6 +292,15 @@ export interface RelateParams<T = any> {
* (`getRelations({ verb, subtype })`) and aggregation (`groupBy:['subtype']`). * (`getRelations({ verb, subtype })`) and aggregation (`groupBy:['subtype']`).
*/ */
subtype?: string subtype?: string
/**
* Visibility tier (reserved top-level field), the verb mirror of `AddParams.visibility`.
* Omit (or pass `'public'`) for normal edges that are counted and returned everywhere.
* Pass `'internal'` for app-internal edges hidden from default `related()` / counts /
* `stats()` but retrievable via `related({ includeInternal: true })`. The `'system'`
* tier is reserved for Brainy's own plumbing and is not accepted here. Stored only when
* not `'public'`.
*/
visibility?: 'public' | 'internal'
/** Connection strength (0-1, default: 1.0) */ /** Connection strength (0-1, default: 1.0) */
weight?: number weight?: number
/** Content for the relationship (optional — overrides auto-computed vector) */ /** Content for the relationship (optional — overrides auto-computed vector) */
@ -282,6 +324,12 @@ export interface UpdateRelationParams<T = any> {
id: string // Relation to update id: string // Relation to update
type?: VerbType // Change verb type type?: VerbType // Change verb type
subtype?: string // Change sub-classification (omit to preserve existing) subtype?: string // Change sub-classification (omit to preserve existing)
/**
* Change the visibility tier. Omit to preserve the existing value. The verb mirror of
* `UpdateParams.visibility`; `'system'` is Brainy-internal — consumer code should only
* use `'public'` / `'internal'`.
*/
visibility?: EntityVisibility
weight?: number // New weight weight?: number // New weight
confidence?: number // New confidence (0-1) confidence?: number // New confidence (0-1)
data?: any // New content data?: any // New content
@ -320,7 +368,24 @@ export interface FindParams<T = any> {
subtype?: string | string[] subtype?: string | string[]
/** Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`) */ /** Metadata filters using BFO operators (e.g., `{ year: { greaterThan: 2020 } }`) */
where?: Partial<T> where?: Partial<T>
// Visibility
/**
* Also return `visibility: 'internal'` entities. By default `find()` returns ONLY
* public entities (those with no `visibility` field, or `visibility: 'public'`);
* app-internal entities are hidden. Set `true` to include them. Applied as a hard
* candidate filter, so `limit` / `offset` stay correct. Does NOT include `'system'`
* entities — use `includeSystem` for those.
*/
includeInternal?: boolean
/**
* Also return `visibility: 'system'` entities (Brainy's own plumbing, e.g. the VFS
* root). Hidden by default and even when `includeInternal` is set. Set `true` only
* when you specifically need to see system entities. Applied as a hard candidate
* filter, so `limit` / `offset` stay correct.
*/
includeSystem?: boolean
// Graph Intelligence // Graph Intelligence
connected?: GraphConstraints connected?: GraphConstraints
@ -481,6 +546,25 @@ export interface GetRelationsParams {
*/ */
subtype?: string | string[] subtype?: string | string[]
/**
* Also return `visibility: 'internal'` relationships.
*
* By default `related()` returns ONLY public relationships (those with no
* `visibility` field, or `visibility: 'public'`); app-internal edges are hidden.
* Set `true` to include them. Applied as a hard candidate filter so `limit` /
* `offset` stay correct. Does NOT include `'system'` edges — use `includeSystem`.
*/
includeInternal?: boolean
/**
* Also return `visibility: 'system'` relationships (Brainy's own plumbing).
*
* Hidden by default and even when `includeInternal` is set. Set `true` only when
* you specifically need system edges. Applied as a hard candidate filter so
* `limit` / `offset` stay correct.
*/
includeSystem?: boolean
/** /**
* Maximum number of results to return * Maximum number of results to return
* *

1
src/utils/paramValidation.ts
View file

@ -534,6 +534,7 @@ export function validateUpdateParams(params: UpdateParams): void {
!params.type && !params.type &&
!params.vector && !params.vector &&
params.subtype === undefined && params.subtype === undefined &&
params.visibility === undefined &&
params.confidence === undefined && params.confidence === undefined &&
params.weight === undefined params.weight === undefined
) { ) {

7
src/utils/rebuildCounts.ts
View file

@ -75,6 +75,11 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
for (const noun of result.items) { for (const noun of result.items) {
const metadata = await storage.getNounMetadata(noun.id) const metadata = await storage.getNounMetadata(noun.id)
if (metadata?.noun) { if (metadata?.noun) {
// 8.0 visibility: the user-facing counts only include public entities.
// Internal/system entities (e.g. the VFS root) are excluded — keeping this
// rebuild consistent with the incremental gating in baseStorage.
const visibility = metadata.visibility
if (visibility === 'internal' || visibility === 'system') continue
const entityType = metadata.noun const entityType = metadata.noun
entityCounts.set(entityType, (entityCounts.get(entityType) || 0) + 1) entityCounts.set(entityType, (entityCounts.get(entityType) || 0) + 1)
totalNouns++ totalNouns++
@ -105,6 +110,8 @@ export async function rebuildCounts(storage: BaseStorage): Promise<RebuildCounts
for (const verb of result.items) { for (const verb of result.items) {
if (verb.verb) { if (verb.verb) {
// 8.0 visibility: exclude internal/system edges from the user-facing counts.
if (verb.visibility === 'internal' || verb.visibility === 'system') continue
const verbType = verb.verb const verbType = verb.verb
verbCounts.set(verbType, (verbCounts.get(verbType) || 0) + 1) verbCounts.set(verbType, (verbCounts.get(verbType) || 0) + 1)
totalVerbs++ totalVerbs++

9
src/vfs/VirtualFileSystem.ts
View file

@ -227,6 +227,9 @@ export class VirtualFileSystem implements IVirtualFileSystem {
console.warn('⚠️ VFS: Root metadata incomplete, repairing...') console.warn('⚠️ VFS: Root metadata incomplete, repairing...')
await this.brain.update({ await this.brain.update({
id: rootId, id: rootId,
// Re-assert system visibility on repair so a pre-8.0 root (created before the
// tier existed) is moved out of the default-visible counts/find() too.
visibility: 'system',
metadata: this.getRootMetadata() metadata: this.getRootMetadata()
}) })
} }
@ -246,6 +249,12 @@ export class VirtualFileSystem implements IVirtualFileSystem {
data: '/', data: '/',
type: NounType.Collection, type: NounType.Collection,
subtype: 'vfs-root', // Standard subtype for the VFS root collection (7.30+) subtype: 'vfs-root', // Standard subtype for the VFS root collection (7.30+)
// visibility 'system' (8.0): the VFS root is Brainy's own plumbing, not user data,
// so it is hidden everywhere by default (getNounCount()/find()/stats()) and surfaces
// only via find({ includeSystem: true }). 'system' is intentionally not part of the
// public AddParams.visibility union ('public' | 'internal') — this is the single
// sanctioned internal setter, hence the cast.
visibility: 'system' as 'public' | 'internal',
metadata: this.getRootMetadata() metadata: this.getRootMetadata()
}) })

233
tests/unit/brainy/visibility.test.ts Normal file
View file

@ -0,0 +1,233 @@
/**
* @module tests/unit/brainy/visibility
* @description Tests for the reserved `visibility` field — the three-tier
* (`public` | `internal` | `system`) gate that controls whether an entity or
* relationship surfaces on default user-facing reads.
*
* Contract under test:
* - Absent === `'public'`: counted and returned everywhere.
* - `'internal'`: hidden from default `find()` / `getRelations()` / counts / `stats()`,
* but retrievable with `includeInternal: true`.
* - `'system'`: Brainy plumbing (the VFS root); hidden everywhere by default,
* surfaced only with `includeSystem: true`. Not settable through the public
* `add()` / `relate()` params (the param type narrows to `'public' | 'internal'`).
* - `visibility` is a reserved top-level field: surfaced top-level on reads, never
* inside `metadata`, and an untyped caller smuggling it through `metadata` is
* normalized (a `'public'`/`'internal'` value is lifted; `'system'` is dropped).
*
* THE key regression: a fresh brain reports `getNounCount() === 0` even though the
* VFS root entity exists — because the root is `visibility: 'system'` and excluded.
*
* Runs under the deterministic embedder (the unit setup sets `BRAINY_UNIT_TEST`).
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../../src/brainy'
import { NounType, VerbType } from '../../../src/types/graphTypes.js'
import { createTestConfig } from '../../helpers/test-factory'
describe('visibility (reserved field)', () => {
let brain: Brainy
beforeEach(async () => {
brain = new Brainy(createTestConfig())
await brain.init()
})
afterEach(async () => {
await brain.close()
})
describe('counts exclude system + internal', () => {
it('a fresh brain reports getNounCount() === 0 (the VFS root is system → excluded)', async () => {
// KEY REGRESSION: the VFS root entity exists after init() but is
// visibility:'system', so it must not show up in the user-facing count.
expect(await brain.getNounCount()).toBe(0)
})
it('add() with no visibility is counted (public default)', async () => {
await brain.add({ type: NounType.Concept, data: 'public thing' })
expect(await brain.getNounCount()).toBe(1)
})
it('add({ visibility: "internal" }) is NOT counted', async () => {
await brain.add({ type: NounType.Concept, data: 'public one' })
await brain.add({ type: NounType.Concept, data: 'app-internal', visibility: 'internal' })
// Only the public entity counts.
expect(await brain.getNounCount()).toBe(1)
})
it('flipping visibility via update() moves the entity in/out of the count', async () => {
const id = await brain.add({ type: NounType.Concept, data: 'flip me' })
expect(await brain.getNounCount()).toBe(1)
await brain.update({ id, visibility: 'internal' })
expect(await brain.getNounCount()).toBe(0)
await brain.update({ id, visibility: 'public' })
expect(await brain.getNounCount()).toBe(1)
})
it('stats().entityCount reports only public entities', async () => {
await brain.add({ type: NounType.Concept, data: 'a' })
await brain.add({ type: NounType.Concept, data: 'b', visibility: 'internal' })
const stats = await brain.stats()
expect(stats.entityCount).toBe(1)
})
})
describe('find() default-excludes internal + system', () => {
it('public entities are returned; internal are hidden by default', async () => {
const pubId = await brain.add({ type: NounType.Concept, data: 'visible' })
await brain.add({ type: NounType.Concept, data: 'hidden', visibility: 'internal' })
const def = await brain.find({ type: NounType.Concept, limit: 100 })
const ids = def.map((r) => r.id)
expect(ids).toContain(pubId)
expect(def.length).toBe(1)
})
it('find({ includeInternal: true }) also returns internal entities', async () => {
const pubId = await brain.add({ type: NounType.Concept, data: 'visible' })
const intId = await brain.add({ type: NounType.Concept, data: 'hidden', visibility: 'internal' })
const withInternal = await brain.find({ type: NounType.Concept, includeInternal: true, limit: 100 })
const ids = withInternal.map((r) => r.id)
expect(ids).toContain(pubId)
expect(ids).toContain(intId)
expect(withInternal.length).toBe(2)
})
it('empty-query find() (no filter) also excludes internal by default', async () => {
const pubId = await brain.add({ type: NounType.Concept, data: 'visible' })
await brain.add({ type: NounType.Concept, data: 'hidden', visibility: 'internal' })
const all = await brain.find({ limit: 100 })
const ids = all.map((r) => r.id)
expect(ids).toContain(pubId)
// Only the one public user entity (VFS root is system → excluded too).
expect(all.length).toBe(1)
})
it('limit stays exact when internal entities are interleaved (hard candidate filter)', async () => {
// Two public + two internal; a default find(limit:2) must return exactly the
// two public ones, not get short-changed by the hidden ones.
await brain.add({ type: NounType.Concept, data: 'pub-1' })
await brain.add({ type: NounType.Concept, data: 'int-1', visibility: 'internal' })
await brain.add({ type: NounType.Concept, data: 'pub-2' })
await brain.add({ type: NounType.Concept, data: 'int-2', visibility: 'internal' })
const page = await brain.find({ type: NounType.Concept, limit: 2 })
expect(page.length).toBe(2)
for (const r of page) {
expect(r.visibility === undefined || r.visibility === 'public').toBe(true)
}
})
})
describe('the VFS root (system) entity', () => {
it('never appears in find(), even with includeInternal', async () => {
const rootId = '00000000-0000-0000-0000-000000000000'
// Sanity: the root really exists (get() is an explicit by-id read, not a default surface).
expect(await brain.get(rootId)).not.toBeNull()
const def = await brain.find({ limit: 1000 })
expect(def.map((r) => r.id)).not.toContain(rootId)
const withInternal = await brain.find({ includeInternal: true, limit: 1000 })
expect(withInternal.map((r) => r.id)).not.toContain(rootId)
})
it('appears only with includeSystem: true', async () => {
const rootId = '00000000-0000-0000-0000-000000000000'
const withSystem = await brain.find({ includeSystem: true, limit: 1000 })
expect(withSystem.map((r) => r.id)).toContain(rootId)
})
it('the root carries visibility "system" when surfaced via get()', async () => {
const rootId = '00000000-0000-0000-0000-000000000000'
const root = await brain.get(rootId)
expect(root?.visibility).toBe('system')
})
})
describe('verbs (relationships) — symmetric to nouns', () => {
it('relate({ visibility: "internal" }) is excluded from getVerbCount() + getRelations() by default, included with includeInternal', async () => {
const a = await brain.add({ type: NounType.Person, data: 'a' })
const b = await brain.add({ type: NounType.Person, data: 'b' })
const c = await brain.add({ type: NounType.Person, data: 'c' })
// One public edge, one internal edge from the same source.
await brain.relate({ from: a, to: b, type: VerbType.RelatedTo })
await brain.relate({ from: a, to: c, type: VerbType.RelatedTo, visibility: 'internal' })
// Counts: only the public edge.
expect(await brain.getVerbCount()).toBe(1)
// getRelations() default: only the public edge.
const def = await brain.getRelations({ from: a })
expect(def.length).toBe(1)
expect(def[0].to).toBe(b)
// getRelations({ includeInternal }): both edges.
const withInternal = await brain.getRelations({ from: a, includeInternal: true })
expect(withInternal.length).toBe(2)
expect(withInternal.map((r) => r.to).sort()).toEqual([b, c].sort())
})
})
describe('visibility is a reserved top-level field', () => {
it('is surfaced top-level on get(), never inside metadata', async () => {
const id = await brain.add({
type: NounType.Concept,
data: 'x',
visibility: 'internal',
metadata: { kind: 'note' }
})
const entity = await brain.get(id)
expect(entity?.visibility).toBe('internal')
// metadata holds ONLY custom fields, never the reserved visibility key.
expect(entity?.metadata).toEqual({ kind: 'note' })
expect((entity?.metadata as Record<string, unknown>)?.visibility).toBeUndefined()
})
it('a public-default entity stores no visibility (absent === public)', async () => {
const id = await brain.add({ type: NounType.Concept, data: 'plain' })
const entity = await brain.get(id)
// Absent — not the literal string 'public'. Kept lean on the common path.
expect(entity?.visibility).toBeUndefined()
})
it('an untyped caller passing visibility inside metadata is normalized (lifted to top-level)', async () => {
// Simulate a JavaScript caller smuggling the reserved key past the compile-time guard.
const id = await brain.add({
type: NounType.Concept,
data: 'y',
metadata: { visibility: 'internal', tag: 't' } as object
})
const entity = await brain.get(id)
// Lifted to the top-level field…
expect(entity?.visibility).toBe('internal')
// …and stripped from the metadata bag.
expect((entity?.metadata as Record<string, unknown>)?.visibility).toBeUndefined()
expect((entity?.metadata as Record<string, unknown>)?.tag).toBe('t')
// It is excluded from the default count, exactly like a top-level internal write.
expect(await brain.getNounCount()).toBe(0)
})
it('a "system" value smuggled through metadata is dropped, not honored', async () => {
// 'system' is Brainy-only; an untyped caller must not be able to set it.
const id = await brain.add({
type: NounType.Concept,
data: 'z',
metadata: { visibility: 'system' } as object
})
const entity = await brain.get(id)
// The smuggled 'system' was dropped → entity stays public (counted, visible).
expect(entity?.visibility).toBeUndefined()
expect(await brain.getNounCount()).toBe(1)
const found = await brain.find({ type: NounType.Concept, limit: 10 })
expect(found.map((r) => r.id)).toContain(id)
})
})
})