From b6beb7f96a765b5c13fdbcaf481a826931426be9 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Mon, 29 Jun 2026 16:05:03 -0700 Subject: [PATCH] perf(8.0): drop O(N)-resident id-keyed storage caches; source counts from the record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The storage layer held five id-keyed Maps (nounTypeByIdCache + the subtype and visibility caches) resident for the writer's lifetime — one entry per live entity, present even when a native provider is registered (it swaps the indexes, not the storage adapter). At billion scale that is hundreds of GB of writer RAM, breaking the "nothing O(N)-resident" invariant. Eliminate all five. Per-type/subtype counts are attributed at the metadata-save site where the type is already in hand, and the delete path re-derives the prior values by reading the record before removing it. O(1) writer RAM on both the native and standalone paths; the latent rebuildTypeCounts staleness edge is gone by construction. Unit 1718 + integration 607 green. --- src/storage/baseStorage.ts | 326 ++++++++++++------------------------- 1 file changed, 102 insertions(+), 224 deletions(-) diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 353be9e4..45990130 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -16,7 +16,6 @@ import { HNSWVerbWithMetadata, StatisticsData } from '../coreTypes.js' -import type { EntityVisibility } from '../coreTypes.js' import { BaseStorageAdapter } from './adapters/baseStorageAdapter.js' import { validateNounType, validateVerbType } from '../utils/typeValidation.js' import { @@ -208,18 +207,6 @@ function getVerbMetadataPath(id: string): string { return `entities/verbs/${shard}/${id}/metadata.json` } -/** - * Extract the entity id from an ID-first metadata path. Both noun and verb - * metadata live at `entities////metadata.json`, so the id is - * the second-to-last path segment. - * @param path - A metadata.json path produced by `getNounMetadataPath` / `getVerbMetadataPath`. - * @returns The id segment, or `undefined` if the path doesn't have the expected shape. - */ -function idFromMetadataPath(path: string): string | undefined { - const segments = path.split('/') - return segments[segments.length - 2] -} - /** * Optional count capabilities probed via duck typing by getNouns()/getVerbs(). * Adapters with a native O(1) count API may implement these; they are not part @@ -322,54 +309,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { */ protected verbSubtypeCountsByType = new Map>() - /** - * In-memory map from noun id → its `noun` (NounType) value, populated when - * `saveNounMetadata_internal()` runs. `saveNoun_internal()` consults this - * to attribute the entity to the correct slot in `nounCountsByType` so the - * persisted `_system/type-statistics.json` is honest. - * - * History: a previous cache was removed in commit `42ae5be` and replaced - * with a hardcoded `return 'thing'` from `getNounType()`, which silently - * poisoned the on-disk type statistics. This cache restores the correct - * behavior. Memory footprint is one Map entry per live noun id (typically - * tens of bytes each); the writer process keeps it for the duration of - * its lifetime and prunes on delete. - */ - protected nounTypeByIdCache = new Map() - - /** - * In-memory map from noun id → its `subtype` string (when set). Parallel to - * `nounTypeByIdCache`. Lets `deleteNounMetadata()` decrement the correct - * subtype bucket without re-reading metadata. Sparse — only ids with a - * non-empty subtype get an entry. - */ - protected nounSubtypeByIdCache = new Map() - - /** - * In-memory map from verb id → `{ verb, subtype }` pair. Verb-side mirror of - * `nounTypeByIdCache` + `nounSubtypeByIdCache`. Lets `deleteVerbMetadata` and - * `updateRelation` decrement the right bucket without a re-read of metadata. - * Sparse — only ids with a non-empty subtype get an entry. - */ - protected verbSubtypeByIdCache = new Map() - // Total: 676 bytes (99.2% reduction vs Map-based tracking) - - /** - * In-memory map from noun id → its non-public `visibility` tier ('internal' | - * 'system'). Parallel to `nounTypeByIdCache`. Lets `saveNoun_internal()` - * (which has no metadata access in its signature) skip the user-facing - * `nounCountsByType` increment for hidden entities, and lets - * `deleteNounMetadata()` skip the matching decrement — keeping the counts - * symmetric. Sparse by design: public entities (the common case) get NO - * entry, so the absence of an id means "public, counted". - */ - protected nounVisibilityByIdCache = new Map() - - /** - * In-memory map from verb id → its non-public `visibility` tier. Verb-side - * mirror of `nounVisibilityByIdCache`. Sparse — public edges get no entry. - */ - protected verbVisibilityByIdCache = new Map() + // Count attribution (type / subtype / visibility) is sourced directly from the + // canonical metadata RECORD, never from id-keyed in-memory caches. The metadata + // save/delete paths already read the prior record (`existingMetadata` on write, + // read-before-delete on remove), so the entity's `noun`/`verb` type, `subtype`, + // and `visibility` are in hand exactly where a count must change — there is no + // need for a parallel O(N) `id → type/subtype/visibility` map resident on the + // writer. The five such caches that used to live here were removed in the 8.0 + // billion-scale RAM pass; `nounCountsByType` / `verbCountsByType` (fixed + // type-indexed Uint32Arrays) and `subtypeCountsByType` / `verbSubtypeCountsByType` + // (bounded by distinct subtype labels) remain because they are NOT id-keyed. // Type caches REMOVED - ID-first paths eliminate need for type lookups! // With ID-first architecture, we construct paths directly from IDs: {SHARD}/{ID}/metadata.json @@ -1037,15 +986,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Reset and reload every piece of adapter-internal derived state after the * underlying objects changed wholesale (restore-from-snapshot): the - * write-through cache, the id→type/subtype caches, type/subtype statistics, - * total counts, and the graph-index singleton (invalidated so the next - * accessor rebuilds from the restored verbs). + * write-through cache, type/subtype statistics, total counts, and the + * graph-index singleton (invalidated so the next accessor rebuilds from the + * restored verbs). Count attribution reads the canonical metadata record, so + * there are no id-keyed caches to clear here. */ protected async reloadDerivedState(): Promise { this.clearWriteCache() - this.nounTypeByIdCache.clear() - this.nounSubtypeByIdCache.clear() - this.verbSubtypeByIdCache.clear() this.nounCountsByType.fill(0) this.verbCountsByType.fill(0) this.subtypeCountsByType.clear() @@ -2633,19 +2580,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Save the metadata (write-cache coherent canonical write) await this.writeCanonicalObject(path, metadata) - // Record the id→type mapping so `saveNoun_internal()` (which receives only - // an HNSWNoun and has no metadata access in its signature) can attribute - // the entity to the right slot in `nounCountsByType`. This is what makes - // `_system/type-statistics.json` honest. Updates the cache even for - // existing entities so a type change via `update()` is reflected. - if (metadata.noun) { - this.nounTypeByIdCache.set(id, metadata.noun as NounType) - } - // Track subtype changes: on type or subtype change via update(), decrement - // the prior bucket before incrementing the new one. Symmetric with the - // delete-path decrement in `deleteNounMetadata()`. - const priorSubtype = this.nounSubtypeByIdCache.get(id) + // the prior bucket before incrementing the new one. The prior (type, subtype) + // comes straight from the canonical record (`existingMetadata`, already loaded + // above) — there is no id-keyed subtype cache. Symmetric with the delete-path + // decrement in `deleteNounMetadata()`. + const priorSubtype = isNew + ? undefined + : (typeof existingMetadata?.subtype === 'string' && (existingMetadata.subtype as string).length > 0 + ? (existingMetadata.subtype as string) + : undefined) const priorTypeForSubtype = isNew ? undefined : (existingMetadata?.noun as NounType | undefined) const newSubtype = typeof metadata.subtype === 'string' && metadata.subtype.length > 0 ? metadata.subtype as string @@ -2657,24 +2601,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { } if (newSubtype && newType && (isNew || priorSubtype !== newSubtype || priorTypeForSubtype !== newType)) { this.incrementSubtypeCount(newType, newSubtype) - this.nounSubtypeByIdCache.set(id, newSubtype) - } else if (!newSubtype && priorSubtype) { - // Subtype cleared by an update — drop the cache entry (decrement already done above) - this.nounSubtypeByIdCache.delete(id) } // Visibility (8.0): only public entities count toward the user-facing totals. - // Warm `nounVisibilityByIdCache` so `saveNoun_internal()` (no metadata access) - // can gate the `nounCountsByType` increment, and `deleteNounMetadata()` can gate - // the matching decrement. Sparse — public ids get no entry. + // The gate reads `metadata.visibility` (new) and `existingMetadata?.visibility` + // (prior) directly off the record — no id-keyed visibility cache. const newVisibility = metadata.visibility const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) const isCounted = isCountedVisibility(newVisibility) - if (isCountedVisibility(newVisibility)) { - this.nounVisibilityByIdCache.delete(id) - } else { - this.nounVisibilityByIdCache.set(id, newVisibility as EntityVisibility) - } // CRITICAL FIX: Increment count for new entities // This runs AFTER metadata is saved, guaranteeing type information is available @@ -2688,11 +2622,21 @@ export abstract class BaseStorage extends BaseStorageAdapter { // used to be bumped unconditionally in saveNoun_internal(), but the HNSW index // re-saves a node on every neighbor-link change, so that inflated the per-type // counts with graph connectivity (e.g. 8 documents could read as 44). - this.nounCountsByType[TypeUtils.getNounIndex(metadata.noun as NounType)]++ + const typeIdx = TypeUtils.getNounIndex(metadata.noun as NounType) + this.nounCountsByType[typeIdx]++ // Persist counts asynchronously (fire and forget) this.scheduleCountPersist().catch(() => { // Ignore persist errors - will retry on next operation }) + // Persist type-statistics on the first entity of a type and every 100th + // thereafter. This trigger used to live in saveNoun_internal(), which had to + // call getNounType() purely to recover the type index; sourcing the type from + // the metadata record here keeps the hot vector-save path free of any type + // lookup. The "only when counted" half of the heuristic holds by construction + // inside this branch. + if (this.nounCountsByType[typeIdx] === 1 || this.nounCountsByType[typeIdx] % 100 === 0) { + await this.saveTypeStatistics() + } } else if (!isNew && metadata.noun && wasCounted !== isCounted) { // Visibility flipped on update(): move the entity in/out of the user-facing // total (counts.json / getNounCount()) AND the per-type counter together, so @@ -2701,6 +2645,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { if (isCounted) { this.incrementEntityCount(metadata.noun) this.nounCountsByType[typeIdx]++ + // Same cadence-gated type-statistics persist as the fresh-add branch — only + // fires when the entity is now counted (public), matching the original + // `counted && (count === 1 || count % 100 === 0)` heuristic. + if (this.nounCountsByType[typeIdx] === 1 || this.nounCountsByType[typeIdx] % 100 === 0) { + await this.saveTypeStatistics() + } } else { this.decrementEntityCount(metadata.noun) if (this.nounCountsByType[typeIdx] > 0) this.nounCountsByType[typeIdx]-- @@ -3027,21 +2977,20 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async deleteNounMetadata(id: string): Promise { await this.ensureInitialized() - // Direct O(1) delete with ID-first path + // Direct O(1) delete with ID-first path. Read the canonical record BEFORE + // removing it: the per-type and subtype decrements are sourced from the + // entity's own metadata (`noun` type, `subtype`, `visibility`) rather than an + // id-keyed cache, keeping type-statistics honest across deletes — symmetric + // with the increments in `saveNounMetadata_internal()`. const path = getNounMetadataPath(id) + const record = await this.readCanonicalObject(path) await this.deleteCanonicalObject(path) - // Prune the id→type cache so a future re-add of the same id (e.g. churn - // during tests) doesn't see a stale type. Lookup the prior type and - // decrement `nounCountsByType` so type-statistics stay honest across - // deletes; symmetric with the increment in `saveNounMetadata_internal()`. - const priorType = this.nounTypeByIdCache.get(id) + const priorType = record?.noun as NounType | undefined // 8.0 visibility: an internal/system entity was never added to `nounCountsByType` - // (gated in `saveNoun_internal()`), so it must not be decremented here either. - const priorCounted = isCountedVisibility(this.nounVisibilityByIdCache.get(id)) - this.nounVisibilityByIdCache.delete(id) + // (gated in `saveNounMetadata_internal()`), so it must not be decremented here either. + const priorCounted = isCountedVisibility(record?.visibility) if (priorType) { - this.nounTypeByIdCache.delete(id) if (priorCounted) { const idx = TypeUtils.getNounIndex(priorType) if (this.nounCountsByType[idx] > 0) { @@ -3049,10 +2998,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { } } - // Symmetric subtype decrement - const priorSubtype = this.nounSubtypeByIdCache.get(id) + // Symmetric subtype decrement — same non-empty-string guard as the write path. + const priorSubtype = typeof record?.subtype === 'string' && (record.subtype as string).length > 0 + ? (record.subtype as string) + : undefined if (priorSubtype) { - this.nounSubtypeByIdCache.delete(id) this.decrementSubtypeCount(priorType, priorSubtype) } } @@ -3108,39 +3058,31 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Save the metadata (write-cache coherent canonical write) await this.writeCanonicalObject(path, metadata) - // Cache verb type for faster lookups - // Track verb subtype changes: on type or subtype change via updateRelation(), - // decrement the prior bucket before incrementing the new one. Symmetric with - // the delete-path decrement in `deleteVerbMetadata()`. - const priorEntry = this.verbSubtypeByIdCache.get(id) + // decrement the prior bucket before incrementing the new one. The prior + // (verb, subtype) is read straight from the canonical record (`existingMetadata`, + // loaded above) — there is no id-keyed verb-subtype cache. Symmetric with the + // delete-path decrement in `deleteVerbMetadata()`. const priorVerbForSubtype = isNew ? undefined : (existingMetadata?.verb as VerbType | undefined) + const priorSubtype = isNew + ? undefined + : (typeof existingMetadata?.subtype === 'string' && (existingMetadata.subtype as string).length > 0 + ? (existingMetadata.subtype as string) + : undefined) const newSubtype = typeof metadata.subtype === 'string' && metadata.subtype.length > 0 ? metadata.subtype as string : undefined - if (priorEntry && (priorEntry.subtype !== newSubtype || priorEntry.verb !== verbType)) { - this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype) - } else if (!priorEntry && priorVerbForSubtype && !isNew) { - // Edge case: cache miss but metadata existed with a subtype (e.g. reader process startup). - const priorSubFromMeta = existingMetadata?.subtype as string | undefined - if (priorSubFromMeta && (priorSubFromMeta !== newSubtype || priorVerbForSubtype !== verbType)) { - this.decrementVerbSubtypeCount(priorVerbForSubtype, priorSubFromMeta) - } + if (priorSubtype && priorVerbForSubtype && (priorSubtype !== newSubtype || priorVerbForSubtype !== verbType)) { + this.decrementVerbSubtypeCount(priorVerbForSubtype, priorSubtype) } - if (newSubtype) { - if (!priorEntry || priorEntry.subtype !== newSubtype || priorEntry.verb !== verbType) { - this.incrementVerbSubtypeCount(verbType, newSubtype) - } - this.verbSubtypeByIdCache.set(id, { verb: verbType, subtype: newSubtype }) - } else if (priorEntry) { - // Subtype cleared by an update — drop the cache entry (decrement done above) - this.verbSubtypeByIdCache.delete(id) + if (newSubtype && (isNew || priorSubtype !== newSubtype || priorVerbForSubtype !== verbType)) { + this.incrementVerbSubtypeCount(verbType, newSubtype) } - // Visibility (8.0): verb mirror of the noun count gating. Warm - // `verbVisibilityByIdCache` so `deleteVerbMetadata()` can prune it. Sparse — - // public edges get no entry. + // Visibility (8.0): verb mirror of the noun count gating. The gate reads + // `metadata.visibility` (new) and `existingMetadata?.visibility` (prior) + // directly off the record — no id-keyed visibility cache. // // NOTE on `verbCountsByType`: unlike the noun path, `saveVerb_internal()` runs // BEFORE this method (relate() saves the verb vector first) and has already done @@ -3150,11 +3092,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { const newVisibility = metadata.visibility const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility) const isCounted = isCountedVisibility(newVisibility) - if (isCounted) { - this.verbVisibilityByIdCache.delete(id) - } else { - this.verbVisibilityByIdCache.set(id, newVisibility as EntityVisibility) - } const verbTypeIdx = TypeUtils.getVerbIndex(verbType) // CRITICAL FIX: Increment verb count for new relationships @@ -3212,19 +3149,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async deleteVerbMetadata(id: string): Promise { await this.ensureInitialized() - // Direct O(1) delete with ID-first path + // Direct O(1) delete with ID-first path. Read the canonical record BEFORE + // removing it so the verb-subtype decrement is sourced from the edge's own + // metadata (`verb` type + `subtype`) rather than an id-keyed cache — symmetric + // with the increment in `saveVerbMetadata_internal()`. Verb deletes do not + // touch `verbCountsByType` in this path (matching prior behavior), so no + // visibility read is needed here. const path = getVerbMetadataPath(id) + const record = await this.readCanonicalObject(path) await this.deleteCanonicalObject(path) - // Symmetric verb subtype decrement - const priorEntry = this.verbSubtypeByIdCache.get(id) - if (priorEntry) { - this.verbSubtypeByIdCache.delete(id) - this.decrementVerbSubtypeCount(priorEntry.verb, priorEntry.subtype) + const priorVerb = record?.verb as VerbType | undefined + const priorSubtype = typeof record?.subtype === 'string' && (record.subtype as string).length > 0 + ? (record.subtype as string) + : undefined + if (priorVerb && priorSubtype) { + this.decrementVerbSubtypeCount(priorVerb, priorSubtype) } - // 8.0 visibility: prune the cache entry (verb deletes don't touch verbCountsByType - // in this path, mirroring the existing verb-subtype delete semantics). - this.verbVisibilityByIdCache.delete(id) // 8.0 MVCC: entity-visible write — advance the generation watermark // (suppressed inside transact batches by the generation store). @@ -3421,7 +3362,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async rebuildSubtypeCounts(): Promise { prodLog.info('[BaseStorage] Rebuilding subtype counts from storage...') this.subtypeCountsByType.clear() - this.nounSubtypeByIdCache.clear() for (let shard = 0; shard < 256; shard++) { const shardHex = shard.toString(16).padStart(2, '0') @@ -3434,10 +3374,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { const metadata = await this.readCanonicalObject(path) if (metadata && metadata.noun && typeof metadata.subtype === 'string' && metadata.subtype.length > 0) { this.incrementSubtypeCount(metadata.noun as NounType, metadata.subtype) - // Path is `entities/nouns///metadata.json` — extract id segment. - const segments = path.split('/') - const idSeg = segments[segments.length - 2] - if (idSeg) this.nounSubtypeByIdCache.set(idSeg, metadata.subtype) } } catch { /* skip unreadable entities */ } } @@ -3538,7 +3474,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { public async rebuildVerbSubtypeCounts(): Promise { prodLog.info('[BaseStorage] Rebuilding verb subtype counts from storage...') this.verbSubtypeCountsByType.clear() - this.verbSubtypeByIdCache.clear() for (let shard = 0; shard < 256; shard++) { const shardHex = shard.toString(16).padStart(2, '0') @@ -3553,9 +3488,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { const verb = metadata.verb as VerbType const subtype = metadata.subtype as string this.incrementVerbSubtypeCount(verb, subtype) - const segments = path.split('/') - const idSeg = segments[segments.length - 2] - if (idSeg) this.verbSubtypeByIdCache.set(idSeg, { verb, subtype }) } } catch { /* skip unreadable verbs */ } } @@ -3660,17 +3592,12 @@ export abstract class BaseStorage extends BaseStorageAdapter { try { const metadata = await this.readCanonicalObject(path) if (metadata && metadata.noun) { - // 8.0 visibility: rebuild only public entities into the user-facing - // per-type stat; warm the cache so later writes/deletes stay symmetric. - const id = idFromMetadataPath(path) + // 8.0 visibility: rebuild only public entities into the user-facing per-type stat. if (isCountedVisibility(metadata.visibility)) { const typeIndex = TypeUtils.getNounIndex(metadata.noun) if (typeIndex >= 0 && typeIndex < NOUN_TYPE_COUNT) { this.nounCountsByType[typeIndex]++ } - if (id) this.nounVisibilityByIdCache.delete(id) - } else if (id) { - this.nounVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) } } } catch (error) { @@ -3697,15 +3624,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { const metadata = await this.readCanonicalObject(path) if (metadata && metadata.verb) { // 8.0 visibility: rebuild only public edges into the user-facing per-type stat. - const id = idFromMetadataPath(path) if (isCountedVisibility(metadata.visibility)) { const typeIndex = TypeUtils.getVerbIndex(metadata.verb) if (typeIndex >= 0 && typeIndex < VERB_TYPE_COUNT) { this.verbCountsByType[typeIndex]++ } - if (id) this.verbVisibilityByIdCache.delete(id) - } else if (id) { - this.verbVisibilityByIdCache.set(id, metadata.visibility as EntityVisibility) } } } catch (error) { @@ -3726,56 +3649,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Resolve the `NounType` for a noun, used to attribute the entity to the - * correct slot in `nounCountsByType` (which backs `_system/type-statistics.json` - * and `brain.stats().entitiesByType`). + * Resolve a noun's `NounType` straight from its canonical metadata record on + * disk. Used by the poisoned-statistics detector (`detectPoisonedTypeStatistics()`) + * to confirm whether on-disk types disagree with a `'thing'`-only persisted + * rollup before triggering a full `rebuildTypeCounts()`. Returns `null` when + * metadata genuinely doesn't exist or the read fails; the caller decides whether + * to skip the entity. There is no in-memory id→type cache — the record is the + * single source of truth. * - * Lookup order: - * 1. `nounTypeByIdCache`, populated by `saveNounMetadata_internal()` - * whenever a noun's metadata is written. This is the common path — - * add() saves metadata before the HNSW noun, so by the time we get - * here the cache is warm. - * 2. `'thing'` as a final fallback when metadata genuinely doesn't exist - * (a noun added without metadata — irregular but tolerated for back- - * compat). Logged so a real bug doesn't go unnoticed. - * - * Note this is intentionally synchronous because `saveNoun_internal()` and - * its callers are not async-friendly at this layer. For cases where only - * an id is known after a process restart and the cache is cold, - * `getNounTypeFromStorageAsync()` is available for callers that can await. - */ - protected getNounType(noun: HNSWNoun): NounType { - const cached = this.nounTypeByIdCache.get(noun.id) - if (cached) return cached - // Cache miss on a noun we're about to write means metadata was not saved - // first. Brainy.add() always saves metadata before HNSW, so this only - // fires in unusual code paths (raw `storage.saveNoun()` without metadata). - prodLog.warn( - `[BaseStorage] getNounType: no type cached for noun ${noun.id}. ` + - `Type-statistics will attribute this entity to 'thing'. ` + - `Call saveNounMetadata() before saveNoun() to avoid stat drift.` - ) - return 'thing' - } - - /** - * Async variant of `getNounType()` that consults disk if the in-memory - * cache is cold (e.g. after a process restart with deferred work). Used by - * `rebuildTypeCounts()` and other callers that can await an IO. Returns - * `null` if metadata genuinely doesn't exist; callers decide whether to - * fall back to 'thing' or skip the entity. + * @param id - The noun id whose type to resolve. + * @returns The stored `NounType`, or `null` if absent/unreadable. */ protected async getNounTypeFromStorageAsync(id: string): Promise { - const cached = this.nounTypeByIdCache.get(id) - if (cached) return cached try { const metadataPath = getNounMetadataPath(id) const metadata = await this.readCanonicalObject(metadataPath) if (metadata && (metadata as NounMetadata).noun) { - const type = (metadata as NounMetadata).noun as NounType - // Warm the cache so subsequent sync calls hit fast. - this.nounTypeByIdCache.set(id, type) - return type + return (metadata as NounMetadata).noun as NounType } } catch { // Storage error — treat as unknown. @@ -3879,28 +3769,16 @@ export abstract class BaseStorage extends BaseStorageAdapter { * Save a noun to storage (ID-first path) */ protected async saveNoun_internal(noun: HNSWNoun): Promise { - const type = this.getNounType(noun) const path = getNounVectorPath(noun.id) - // Per-type stats counter (`nounCountsByType`, read by stats().entitiesByType) - // is maintained in saveNounMetadata_internal(), gated on isNew + visibility — - // NOT here. saveNoun_internal() also runs on HNSW neighbor-link re-saves, so - // incrementing here inflated the per-type counts with graph connectivity. We - // only READ the (externally-maintained) count below to decide when to persist. - const typeIndex = TypeUtils.getNounIndex(type) - const counted = isCountedVisibility(this.nounVisibilityByIdCache.get(noun.id)) - - // Write-cache coherent canonical write + // Hot path: write the vector record only. Per-type counters + // (`nounCountsByType`, read by stats().entitiesByType) AND the periodic + // type-statistics persist trigger are both maintained in + // saveNounMetadata_internal(), which holds the canonical metadata record — + // and therefore the NounType + visibility — at the exact point a count + // changes. saveNoun_internal() also re-runs on every HNSW neighbor-link + // re-save, so it deliberately performs NO type lookup and NO count work here. await this.writeCanonicalObject(path, noun) - - // Periodically save statistics - // Also save on first noun of each type to ensure low-count types are tracked - const shouldSave = counted && - (this.nounCountsByType[typeIndex] === 1 || // First noun of type - this.nounCountsByType[typeIndex] % 100 === 0) // Every 100th - if (shouldSave) { - await this.saveTypeStatistics() - } } /**