soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): visibility field (public/internal/system) on nouns + verbs

Browse source 
Adds a reserved, top-level `visibility` field (mirrors the subtype rollout):
'public' (default, surfaced) | 'internal' (developer app-internal — hidden from
default find/count/stats, opt-in via includeInternal) | 'system' (Brainy
plumbing, library-set only).

Fixes a real leak: the VFS root entity counted in getNounCount() and appeared in
find() (a fresh brain reported 1 entity). It is now visibility:'system' →
excluded from every user-facing surface. Developers also get a first-class
hidden-unless-asked tier (e.g. learned internals vs user-exposed data).

- Reserved (RESERVED_ENTITY_FIELDS / RESERVED_RELATION_FIELDS) — spoof-proof from metadata.
- Threaded through add/relate/update/transact; surfaced top-level on reads.
- Default exclusion in counts (baseStorage), find()/related() (hard candidate
  filter via excludeVisibility — keeps topK/limit correct), and stats;
  includeInternal/includeSystem opt-ins.
- VFS root marked 'system'.

Tests: visibility.test.ts 17/17 (fresh-brain getNounCount()===0, internal hidden
+ opt-in, verb symmetry, top-level surfacing, metadata-spoof rejection). Unit
1431 green; count-synchronization integration now passes (off-by-one fixed).
This commit is contained in:
David Snelling 2026-06-16 15:20:26 -07:00
parent 0ca0e5c6cc
commit f4dea80176
10 changed files with 777 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

191
src/storage/baseStorage.ts
View file

@ -16,6 +16,7 @@ import {
HNSWVerbWithMetadata,
StatisticsData
} from '../coreTypes.js'
import type { EntityVisibility } from '../coreTypes.js'
import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js'
import { validateNounType, validateVerbType } from '../utils/typeValidation.js'
import {
@ -192,6 +193,18 @@ function getVerbMetadataPath(id: string): string {
return `entities/verbs/${shard}/${id}/metadata.json`
}
/**
* Extract the entity id from an ID-first metadata path. Both noun and verb
* metadata live at `entities/<kind>/<shard>/<id>/metadata.json`, so the id is
* the second-to-last path segment.
* @param path - A metadata.json path produced by `getNounMetadataPath` / `getVerbMetadataPath`.
* @returns The id segment, or `undefined` if the path doesn't have the expected shape.
*/
function idFromMetadataPath(path: string): string | undefined {
const segments = path.split('/')
return segments[segments.length - 2]
}
/**
* Optional count capabilities probed via duck typing by getNouns()/getVerbs().
* Adapters with a native O(1) count API may implement these; they are not part
@ -213,6 +226,19 @@ interface OptionalCountCapabilities {
}) => Promise<number>
}
/**
* Whether an entity/relationship with the given visibility tier counts toward
* the user-facing counts (`counts.json` totals, `nounCountsByType`, `stats()`).
* Public entities (absent tier === `'public'`) are counted; `'internal'` and
* `'system'` are excluded. Single source of truth for the count-exclusion rule.
*
* @param visibility - The stored visibility value (may be `undefined`/`unknown`).
* @returns `true` when the entity should be counted, `false` for internal/system.
*/
function isCountedVisibility(visibility: unknown): boolean {
return visibility !== 'internal' && visibility !== 'system'
}
/**
* Base storage adapter that implements common functionality
* This is an abstract class that should be extended by specific storage adapters
@ -313,6 +339,23 @@ export abstract class BaseStorage extends BaseStorageAdapter {
protected verbSubtypeByIdCache = new Map<string, { verb: VerbType; subtype: string }>()
// Total: 676 bytes (99.2% reduction vs Map-based tracking)
/**
* In-memory map from noun id → its non-public `visibility` tier ('internal' |
* 'system'). Parallel to `nounTypeByIdCache`. Lets `saveNoun_internal()`
* (which has no metadata access in its signature) skip the user-facing
* `nounCountsByType` increment for hidden entities, and lets
* `deleteNounMetadata()` skip the matching decrement — keeping the counts
* symmetric. Sparse by design: public entities (the common case) get NO
* entry, so the absence of an id means "public, counted".
*/
protected nounVisibilityByIdCache = new Map<string, EntityVisibility>()
/**
* In-memory map from verb id → its non-public `visibility` tier. Verb-side
* mirror of `nounVisibilityByIdCache`. Sparse — public edges get no entry.
*/
protected verbVisibilityByIdCache = new Map<string, EntityVisibility>()
// Type caches REMOVED - ID-first paths eliminate need for type lookups!
// With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json
// Type is just a field in the metadata, indexed by MetadataIndexManager for queries
@ -1652,6 +1695,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
: [subtypeFilterValue]
)
: null
// 8.0 visibility exclusion — applied after metadata load (verb visibility lives in
// metadata), so hidden edges are skipped BEFORE the pagination window fills.
const excludeVisibility = (filter as { excludeVisibility?: string[] } | undefined)?.excludeVisibility
const filterExcludeVisibility = excludeVisibility && excludeVisibility.length > 0
? new Set(excludeVisibility)
: null
// Iterate by shards (0x00-0xFF) instead of types - single pass!
for (let shard = 0; shard < 256 && collectedVerbs.length < peekCount; shard++) {
@ -1698,6 +1747,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
}
}
// Apply visibility exclusion (8.0). Absent === 'public' (kept); a stored
// 'internal'/'system' value is dropped when its tier is excluded.
if (filterExcludeVisibility) {
const visibility = metadata?.visibility as string | undefined
if (visibility && filterExcludeVisibility.has(visibility)) {
continue
}
}
// Combine verb + metadata via the canonical hydration helper —
// reserved fields top-level, ONLY custom fields in `metadata`
// (this site previously echoed the full flat record).
@ -1744,6 +1802,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
targetId?: string | string[]
service?: string | string[]
metadata?: Record<string, any>
/**
* 8.0 visibility: tiers to exclude from results (e.g. `['internal','system']`).
* Applied after metadata load in the full scan, so it disqualifies the
* metadata-less graph-index fast paths (same as `subtype`).
*/
excludeVisibility?: string[]
}
}): Promise<{
items: HNSWVerbWithMetadata[]
@ -1764,10 +1828,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// fallthrough to the full shard scan (which loads metadata and applies the
// subtype check after the load). Subtype is not stored on the raw HNSWVerb;
// it's a metadata field. The graph-index fast paths return verbs without
// metadata, so they can't apply subtype filtering correctly.
// metadata, so they can't apply subtype filtering correctly. The 8.0
// `excludeVisibility` filter (also a metadata field) disqualifies them the same way.
if (
options?.filter &&
!(options.filter as { verbType?: string | string[]; subtype?: string | string[] }).subtype
!(options.filter as { verbType?: string | string[]; subtype?: string | string[] }).subtype &&
!(options.filter as { excludeVisibility?: string[] }).excludeVisibility
) {
// CRITICAL VFS FIX: If filtering by sourceId + verbType (most common VFS pattern!)
// This is the query PathResolver.getChildren() uses: related({ from: dirId, type: VerbType.Contains })
@ -2367,16 +2433,40 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.nounSubtypeByIdCache.delete(id)
}
// Visibility (8.0): only public entities count toward the user-facing totals.
// Warm `nounVisibilityByIdCache` so `saveNoun_internal()` (no metadata access)
// can gate the `nounCountsByType` increment, and `deleteNounMetadata()` can gate
// the matching decrement. Sparse — public ids get no entry.
const newVisibility = metadata.visibility
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
const isCounted = isCountedVisibility(newVisibility)
if (isCountedVisibility(newVisibility)) {
this.nounVisibilityByIdCache.delete(id)
} else {
this.nounVisibilityByIdCache.set(id, newVisibility as EntityVisibility)
}
// CRITICAL FIX: Increment count for new entities
// This runs AFTER metadata is saved, guaranteeing type information is available
// Uses synchronous increment since storage operations are already serialized
// Fixes Bug #1: Count synchronization failure during add() and import()
if (isNew && metadata.noun) {
// 8.0: skip the user-facing total for internal/system entities (counts.json + getNounCount()).
if (isNew && metadata.noun && isCounted) {
this.incrementEntityCount(metadata.noun)
// Persist counts asynchronously (fire and forget)
this.scheduleCountPersist().catch(() => {
// Ignore persist errors - will retry on next operation
})
} else if (!isNew && metadata.noun && wasCounted !== isCounted) {
// Visibility flipped on update(): move the entity in/out of the user-facing total
// (counts.json / getNounCount()). `nounCountsByType` is gated independently in
// `saveNoun_internal()` off the cache warmed just above, so it is not touched here.
if (isCounted) {
this.incrementEntityCount(metadata.noun)
} else {
this.decrementEntityCount(metadata.noun)
}
this.scheduleCountPersist().catch(() => {})
}
// 8.0 MVCC: entity-visible write — advance the generation watermark
@ -2707,11 +2797,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// decrement `nounCountsByType` so type-statistics stay honest across
// deletes; symmetric with the increment in `saveNounMetadata_internal()`.
const priorType = this.nounTypeByIdCache.get(id)
// 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
// (gated in `saveNoun_internal()`), so it must not be decremented here either.
const priorCounted = isCountedVisibility(this.nounVisibilityByIdCache.get(id))
this.nounVisibilityByIdCache.delete(id)
if (priorType) {
this.nounTypeByIdCache.delete(id)
const idx = TypeUtils.getNounIndex(priorType)
if (this.nounCountsByType[idx] > 0) {
this.nounCountsByType[idx]--
if (priorCounted) {
const idx = TypeUtils.getNounIndex(priorType)
if (this.nounCountsByType[idx] > 0) {
this.nounCountsByType[idx]--
}
}
// Symmetric subtype decrement
@ -2803,16 +2899,52 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.verbSubtypeByIdCache.delete(id)
}
// Visibility (8.0): verb mirror of the noun count gating. Warm
// `verbVisibilityByIdCache` so `deleteVerbMetadata()` can prune it. Sparse —
// public edges get no entry.
//
// NOTE on `verbCountsByType`: unlike the noun path, `saveVerb_internal()` runs
// BEFORE this method (relate() saves the verb vector first) and has already done
// an UNCONDITIONAL `verbCountsByType[idx]++`. We therefore COMPENSATE here: for a
// new hidden edge, undo that bump. `updateRelation()` does not re-run
// `saveVerb_internal()`, so on a visibility flip we adjust the bucket directly.
const newVisibility = metadata.visibility
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
const isCounted = isCountedVisibility(newVisibility)
if (isCounted) {
this.verbVisibilityByIdCache.delete(id)
} else {
this.verbVisibilityByIdCache.set(id, newVisibility as EntityVisibility)
}
const verbTypeIdx = TypeUtils.getVerbIndex(verbType)
// CRITICAL FIX: Increment verb count for new relationships
// This runs AFTER metadata is saved
// Uses synchronous increment since storage operations are already serialized
// Fixes Bug #2: Count synchronization failure during relate() and import()
// 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
if (isNew) {
this.incrementVerbCount(verbType)
if (isCounted) {
this.incrementVerbCount(verbType)
} else {
// Hidden edge: undo the unconditional bump from saveVerb_internal().
if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]--
}
// Persist counts asynchronously (fire and forget)
this.scheduleCountPersist().catch(() => {
// Ignore persist errors - will retry on next operation
})
} else if (wasCounted !== isCounted) {
// Visibility flipped on updateRelation() (saveVerb_internal did not run): move the
// edge in/out of both the user-facing total and the per-type bucket.
if (isCounted) {
this.incrementVerbCount(verbType)
this.verbCountsByType[verbTypeIdx]++
} else {
this.decrementVerbCount(verbType)
if (this.verbCountsByType[verbTypeIdx] > 0) this.verbCountsByType[verbTypeIdx]--
}
this.scheduleCountPersist().catch(() => {})
}
// 8.0 MVCC: entity-visible write — advance the generation watermark
@ -2855,6 +2987,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
this.verbSubtypeByIdCache.delete(id)
this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype)
}
// 8.0 visibility: prune the cache entry (verb deletes don't touch verbCountsByType
// in this path, mirroring the existing verb-subtype delete semantics).
this.verbVisibilityByIdCache.delete(id)
// 8.0 MVCC: entity-visible write — advance the generation watermark
// (suppressed inside transact batches by the generation store).
@ -3290,9 +3425,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
try {
const metadata = await this.readCanonicalObject(path)
if (metadata && metadata.noun) {
const typeIndex = TypeUtils.getNounIndex(metadata.noun)
if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) {
this.nounCountsByType[typeIndex]++
// 8.0 visibility: rebuild only public entities into the user-facing
// per-type stat; warm the cache so later writes/deletes stay symmetric.
const id = idFromMetadataPath(path)
if (isCountedVisibility(metadata.visibility)) {
const typeIndex = TypeUtils.getNounIndex(metadata.noun)
if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) {
this.nounCountsByType[typeIndex]++
}
if (id) this.nounVisibilityByIdCache.delete(id)
} else if (id) {
this.nounVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility)
}
}
} catch (error) {
@ -3318,9 +3461,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
try {
const metadata = await this.readCanonicalObject(path)
if (metadata && metadata.verb) {
const typeIndex = TypeUtils.getVerbIndex(metadata.verb)
if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) {
this.verbCountsByType[typeIndex]++
// 8.0 visibility: rebuild only public edges into the user-facing per-type stat.
const id = idFromMetadataPath(path)
if (isCountedVisibility(metadata.visibility)) {
const typeIndex = TypeUtils.getVerbIndex(metadata.verb)
if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) {
this.verbCountsByType[typeIndex]++
}
if (id) this.verbVisibilityByIdCache.delete(id)
} else if (id) {
this.verbVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility)
}
}
} catch (error) {
@ -3497,17 +3647,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
const type = this.getNounType(noun)
const path = getNounVectorPath(noun.id)
// Update type tracking
// Update type tracking — but only for entities that count toward the
// user-facing per-type stats. `nounVisibilityByIdCache` (warmed by
// `saveNounMetadata_internal()`) flags internal/system ids; public ids are
// absent, so the common case still increments. 8.0 visibility exclusion.
const typeIndex = TypeUtils.getNounIndex(type)
this.nounCountsByType[typeIndex]++
const counted = isCountedVisibility(this.nounVisibilityByIdCache.get(noun.id))
if (counted) {
this.nounCountsByType[typeIndex]++
}
// Write-cache coherent canonical write
await this.writeCanonicalObject(path, noun)
// Periodically save statistics
// Also save on first noun of each type to ensure low-count types are tracked
const shouldSave = this.nounCountsByType[typeIndex] === 1 || // First noun of type
this.nounCountsByType[typeIndex] % 100 === 0 // Every 100th
const shouldSave = counted &&
(this.nounCountsByType[typeIndex] === 1 || // First noun of type
this.nounCountsByType[typeIndex] % 100 === 0) // Every 100th
if (shouldSave) {
await this.saveTypeStatistics()
}