soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): visibility field (public/internal/system) on nouns + verbs

Browse source 
Adds a reserved, top-level `visibility` field (mirrors the subtype rollout):
'public' (default, surfaced) | 'internal' (developer app-internal — hidden from
default find/count/stats, opt-in via includeInternal) | 'system' (Brainy
plumbing, library-set only).

Fixes a real leak: the VFS root entity counted in getNounCount() and appeared in
find() (a fresh brain reported 1 entity). It is now visibility:'system' →
excluded from every user-facing surface. Developers also get a first-class
hidden-unless-asked tier (e.g. learned internals vs user-exposed data).

- Reserved (RESERVED_ENTITY_FIELDS / RESERVED_RELATION_FIELDS) — spoof-proof from metadata.
- Threaded through add/relate/update/transact; surfaced top-level on reads.
- Default exclusion in counts (baseStorage), find()/related() (hard candidate
  filter via excludeVisibility — keeps topK/limit correct), and stats;
  includeInternal/includeSystem opt-ins.
- VFS root marked 'system'.

Tests: visibility.test.ts 17/17 (fresh-brain getNounCount()===0, internal hidden
+ opt-in, verb symmetry, top-level surfacing, metadata-spoof rejection). Unit
1431 green; count-synchronization integration now passes (off-by-one fixed).
This commit is contained in:
David Snelling 2026-06-16 15:20:26 -07:00
parent 0ca0e5c6cc
commit f4dea80176
10 changed files with 777 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

179
src/brainy.ts
View file

@ -14,7 +14,7 @@ import type { StorageOptions } from './storage/storageFactory.js'
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
import { BaseStorage } from './storage/baseStorage.js'
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
import type { HNSWNounWithMetadata, HNSWVerbWithMetadata } from './coreTypes.js'
import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
import {
defaultEmbeddingFunction,
cosineDistance,
@ -1248,6 +1248,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
data: params.data,
noun: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
// visibility: stored only when not 'public' (absent === public, keeps records lean)
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
service: params.service,
createdAt: Date.now(),
updatedAt: Date.now(),
@ -1269,6 +1272,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
level: 0,
type: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.weight !== undefined && { weight: params.weight }),
createdAt: Date.now(),
@ -1542,6 +1547,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Flatten common entity fields to top level
type: entity.type,
subtype: entity.subtype,
visibility: entity.visibility,
metadata: entity.metadata,
data: entity.data,
confidence: entity.confidence,
@ -1572,6 +1578,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
vector: noun.vector,
type: noun.type || NounType.Thing,
subtype: noun.subtype,
visibility: noun.visibility,
// Standard fields at top-level
confidence: noun.confidence,
@ -1622,6 +1629,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
vector: [], // Stub vector (empty array - vectors not loaded for metadata-only)
type: (reserved.noun as NounType) || NounType.Thing,
subtype: reserved.subtype as string | undefined,
visibility: reserved.visibility as EntityVisibility | undefined,
// Standard fields from metadata
confidence: reserved.confidence as number | undefined,
@ -1703,6 +1711,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
: 'nothing — revisions are system-managed'
)
}
// 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal'
// value IS user-settable and is silently remapped to the top-level param below.)
if (reserved.visibility === 'system') {
this.warnDroppedReservedField(method, 'visibility', "the 'visibility' param ('public' | 'internal')")
}
if (method === 'update') {
if (reserved.service !== undefined) {
this.warnDroppedReservedField(method, 'service', 'nothing — fixed at add() time')
@ -1750,6 +1763,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
typeof reserved.weight === 'number' && { weight: reserved.weight }),
...(params.subtype === undefined &&
typeof reserved.subtype === 'string' && { subtype: reserved.subtype }),
...(params.visibility === undefined &&
(reserved.visibility === 'public' || reserved.visibility === 'internal') && {
visibility: reserved.visibility as 'public' | 'internal'
}),
...(params.service === undefined &&
typeof reserved.service === 'string' && { service: reserved.service }),
...(params.createdBy === undefined &&
@ -1828,6 +1845,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
if (reserved._rev !== undefined) {
this.warnDroppedReservedField(method, '_rev', 'nothing — system-managed')
}
// 'system' visibility is Brainy-only — a user bag cannot set it. (A 'public'/'internal'
// value IS user-settable and is silently remapped to the top-level param below.)
if (reserved.visibility === 'system') {
this.warnDroppedReservedField(method, 'visibility', "the 'visibility' param ('public' | 'internal')")
}
if (method === 'updateRelation' && reserved.service !== undefined) {
this.warnDroppedReservedField(method, 'service', 'nothing — fixed at relate() time')
}
@ -1861,6 +1883,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
typeof reserved.weight === 'number' && { weight: reserved.weight }),
...(params.subtype === undefined &&
typeof reserved.subtype === 'string' && { subtype: reserved.subtype }),
...(params.visibility === undefined &&
(reserved.visibility === 'public' || reserved.visibility === 'internal') && {
visibility: reserved.visibility as 'public' | 'internal'
}),
...(params.service === undefined &&
typeof reserved.service === 'string' && { service: reserved.service })
}
@ -1894,7 +1920,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.weight === undefined &&
typeof reserved.weight === 'number' && { weight: reserved.weight }),
...(params.subtype === undefined &&
typeof reserved.subtype === 'string' && { subtype: reserved.subtype })
typeof reserved.subtype === 'string' && { subtype: reserved.subtype }),
...(params.visibility === undefined &&
(reserved.visibility === 'public' || reserved.visibility === 'internal') && {
visibility: reserved.visibility as 'public' | 'internal'
})
}
}
@ -2043,7 +2073,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }),
// Update subtype if provided, otherwise preserve existing
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype })
...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }),
// Visibility: take the new value if provided, else preserve existing. Stored only
// when the effective value is not 'public' (absent === public, keeps records lean).
// A change to 'public' therefore drops the field entirely.
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existing.visibility
})
}
// Build entity structure for metadata index (with top-level fields)
@ -2054,6 +2090,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
level: 0,
type: params.type || existing.type,
subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existing.visibility
}),
confidence: params.confidence !== undefined ? params.confidence : existing.confidence,
weight: params.weight !== undefined ? params.weight : existing.weight,
createdAt: existing.createdAt,
@ -2539,6 +2578,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.metadata || {}),
verb: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
// visibility: stored only when not 'public' (absent === public, keeps records lean)
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
weight: params.weight ?? 1.0,
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.service !== undefined && { service: params.service }),
@ -2555,6 +2597,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
verb: params.type,
type: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
weight: params.weight ?? 1.0,
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.service !== undefined && { service: params.service }),
@ -2751,6 +2795,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.subtype !== undefined
? { subtype: params.subtype }
: existingRec.subtype !== undefined && { subtype: existingRec.subtype }),
// Visibility: new value if provided, else preserve existing; stored only when the
// effective value is not 'public' (a change to 'public' drops the field).
...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existingRec.visibility
}),
weight: params.weight ?? existingRec.weight ?? 1.0,
...(params.confidence !== undefined
? { confidence: params.confidence }
@ -2777,6 +2826,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.subtype !== undefined
? { subtype: params.subtype }
: existingRec.subtype !== undefined && { subtype: existingRec.subtype }),
...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existingRec.visibility
}),
weight: updatedMetadata.weight,
metadata: newMetadata,
data: updatedMetadata.data,
@ -2888,6 +2940,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
filter.service = params.service
}
// Visibility (8.0): exclude hidden tiers by default. The exclusion is applied in the
// storage scan AFTER metadata load (where verb.visibility is known), so the storage
// limit stays exact. Like subtype, it disqualifies the metadata-less fast paths.
const excludedTiers = this.excludedVisibilityTiers(params)
if (excludedTiers) {
filter.excludeVisibility = excludedTiers
}
// VFS relationships are no longer filtered
// VFS is part of the knowledge graph - users can filter explicitly if needed
@ -3538,6 +3598,46 @@ export class Brainy<T = any> implements BrainyInterface<T> {
})
}
/**
* The visibility tiers a read should EXCLUDE, given the caller's opt-ins.
* Default (no opts) hides both `'internal'` and `'system'`; `includeInternal`
* unhides internal; `includeSystem` unhides system. Returns `null` when nothing
* is excluded (both opts set) so callers can skip the filter entirely.
*
* @param params - The read params carrying `includeInternal` / `includeSystem`.
* @returns The set of excluded tier strings, or `null` for "exclude nothing".
*/
private excludedVisibilityTiers(
params: { includeInternal?: boolean; includeSystem?: boolean }
): EntityVisibility[] | null {
const excluded: EntityVisibility[] = []
if (!params.includeInternal) excluded.push('internal')
if (!params.includeSystem) excluded.push('system')
return excluded.length > 0 ? excluded : null
}
/**
* Resolve the set of entity/relationship ids to hide for this read, by asking
* the metadata index for everything tagged with an excluded visibility tier.
* Public entities carry NO stored `visibility` (absent === public), so they are
* never in this set — exactly the entities we want to keep. Returns an empty set
* when nothing is excluded. The result is used as a HARD candidate filter
* (subtracted from candidate ids BEFORE pagination), so `limit`/`offset` stay correct.
*
* @param params - The read params carrying the visibility opt-ins.
* @returns A set of ids to omit from results.
*/
private async resolveHiddenIds(
params: { includeInternal?: boolean; includeSystem?: boolean }
): Promise<Set<string>> {
const excluded = this.excludedVisibilityTiers(params)
if (!excluded) return new Set()
const ids = await this.metadataIndex.getIdsForFilter({
visibility: excluded.length === 1 ? excluded[0] : { oneOf: excluded }
})
return new Set(ids)
}
async find(query: string | FindParams<T>): Promise<Result<T>[]> {
await this.ensureInitialized()
@ -3557,6 +3657,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return this.findAggregate(params)
}
// Visibility (8.0): resolve the ids to hide for this read ONCE. Default hides
// internal + system; opt back in with includeInternal / includeSystem. Applied as a
// hard candidate filter at every candidate-gathering point below so limit/offset stay
// correct. Empty set when both opts are set (nothing excluded).
const hiddenIds = await this.resolveHiddenIds(params)
const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id)
const startTime = Date.now()
const result = await (async () => {
let results: Result<T>[] = []
@ -3632,6 +3739,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
filteredIds = await this.metadataIndex.getIdsForFilter(filter)
}
// Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact.
if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id))
// Paginate BEFORE loading entities (production-scale!)
const limit = params.limit || 10
const offset = params.offset || 0
@ -3658,7 +3768,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Unfiltered sort: column store handles this at O(K log S) scale.
// No per-entity storage reads, no bucketing precision loss.
if (params.orderBy) {
const k = limit + offset
// Over-fetch by the hidden count so dropping hidden ids still fills the page.
const k = limit + offset + hiddenIds.size
const sortedIntIds = await this.metadataIndex.columnStore.sortTopK(
params.orderBy,
params.order || 'asc',
@ -3668,9 +3779,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Convert int IDs (BigInt at the provider boundary) to UUIDs and
// paginate. Number() narrowing is lossless under the u32 guard.
const idMapper = this.metadataIndex.getIdMapper()
const allUuids = sortedIntIds
let allUuids = sortedIntIds
.map(intId => idMapper.getUuid(Number(intId)))
.filter((uuid): uuid is string => uuid !== undefined)
// Visibility hard filter — drop hidden ids BEFORE pagination.
if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id))
const pageIds = allUuids.slice(offset, offset + limit)
const entitiesMap = await this.batchGet(pageIds)
@ -3691,9 +3804,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
filter.isVFSEntity = { ne: true }
}
// Use metadata index if we need to filter
// Use metadata index when there's an actual filter (e.g. excludeVFS). The empty
// filter returns nothing from getIdsForFilter, so the unfiltered case below uses
// getNouns instead (it returns all nouns, including their visibility).
if (Object.keys(filter).length > 0) {
const filteredIds = await this.metadataIndex.getIdsForFilter(filter)
let filteredIds = await this.metadataIndex.getIdsForFilter(filter)
// Visibility hard filter — drop hidden ids BEFORE pagination.
if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id))
const pageIds = filteredIds.slice(offset, offset + limit)
// Batch-load entities for 10x faster cloud storage performance
@ -3705,13 +3822,19 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
}
} else {
// No filtering needed, use direct storage query
// No metadata filter, use direct storage query. Over-fetch by the hidden count
// so the visibility filter below still fills the requested page (limit-exact).
const storageResults = await this.storage.getNouns({
pagination: { limit: limit + offset, offset: 0 }
pagination: { limit: limit + offset + hiddenIds.size, offset: 0 }
})
for (let i = offset; i < Math.min(offset + limit, storageResults.items.length); i++) {
const noun = storageResults.items[i]
// Visibility hard filter on the materialized nouns (each carries `visibility`).
const visible = hiddenIds.size > 0
? storageResults.items.filter((noun) => noun && !hiddenIds.has(noun.id))
: storageResults.items
for (let i = offset; i < Math.min(offset + limit, visible.length); i++) {
const noun = visible[i]
if (noun) {
const entity = await this.convertNounToEntity(noun)
results.push(this.createResult(noun.id, 1.0, entity))
@ -3766,6 +3889,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}
preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter)
// Visibility hard filter — restrict the HNSW candidate set to non-hidden ids.
if (hiddenIds.size > 0) {
preResolvedMetadataIds = preResolvedMetadataIds.filter((id) => !hiddenIds.has(id))
}
// Short-circuit: if metadata filter matches nothing, skip expensive vector search
if (preResolvedMetadataIds.length === 0) {
return []
@ -3829,6 +3957,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
results = Array.from(uniqueResults.values())
}
// Visibility hard filter for vector/text/proximity candidates (the pre-resolved
// path already excluded hidden ids; this covers searches with no metadata filter).
// Applied BEFORE the final sort+slice so limit/offset remain exact.
if (hiddenIds.size > 0 && results.length > 0) {
results = results.filter((r) => !isHidden(r.id))
}
// Apply metadata filtering using pre-resolved IDs (metadata-first optimization).
// When vector search was performed, HNSW already filtered by candidateIds —
// this block handles pagination, entity loading, and metadata-only queries.
@ -5443,6 +5578,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
to: core.targetId,
type: (reserved.verb ?? core.verb) as VerbType,
...(reserved.subtype !== undefined && { subtype: reserved.subtype as string }),
...(reserved.visibility !== undefined && { visibility: reserved.visibility as EntityVisibility }),
weight: (reserved.weight as number) ?? 1.0,
data: reserved.data,
metadata: custom as T,
@ -5754,6 +5890,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
data: params.data,
noun: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
// visibility: stored only when not 'public' (absent === public, keeps records lean)
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
service: params.service,
createdAt: now,
updatedAt: now,
@ -5769,6 +5908,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
level: 0,
type: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.weight !== undefined && { weight: params.weight }),
createdAt: now,
@ -5871,7 +6012,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
existing.weight !== undefined && { weight: existing.weight }),
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.subtype === undefined &&
existing.subtype !== undefined && { subtype: existing.subtype })
existing.subtype !== undefined && { subtype: existing.subtype }),
// Visibility: new value if provided, else preserve existing; stored only when the
// effective value is not 'public' (a change to 'public' drops the field).
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existing.visibility
})
}
const entityForIndexing = {
id: params.id,
@ -5880,6 +6026,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
level: 0,
type: params.type || existing.type,
subtype: params.subtype !== undefined ? params.subtype : existing.subtype,
...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && {
visibility: params.visibility ?? existing.visibility
}),
confidence: params.confidence !== undefined ? params.confidence : existing.confidence,
weight: params.weight !== undefined ? params.weight : existing.weight,
createdAt: existing.createdAt,
@ -6083,6 +6232,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
...(params.metadata || {}),
verb: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
// visibility: stored only when not 'public' (absent === public, keeps records lean)
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
weight: params.weight ?? 1.0,
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.service !== undefined && { service: params.service }),
@ -6097,6 +6249,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
verb: params.type,
type: params.type,
...(params.subtype !== undefined && { subtype: params.subtype }),
...(params.visibility !== undefined &&
params.visibility !== 'public' && { visibility: params.visibility }),
weight: params.weight ?? 1.0,
...(params.confidence !== undefined && { confidence: params.confidence }),
...(params.service !== undefined && { service: params.service }),
@ -10007,6 +10161,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
to: v.targetId,
type: (v.verb || v.type) as VerbType,
...(v.subtype !== undefined && { subtype: v.subtype }),
...(v.visibility !== undefined && { visibility: v.visibility }),
weight: v.weight ?? 1.0,
...(typeof v.confidence === 'number' && { confidence: v.confidence }),
data: v.data,