From cb16a39a0cfe2052f6e9c149624f3933b6b64daa Mon Sep 17 00:00:00 2001 From: David Snelling Date: Tue, 9 Jun 2026 15:33:56 -0700 Subject: [PATCH] =?UTF-8?q?chore(8.0):=20Phase=20A=20+=20B=20=E2=80=94=20p?= =?UTF-8?q?urge=20all=20@deprecated=20APIs=20+=20cacheManager=20dead=20bra?= =?UTF-8?q?nches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PHASE A — every @deprecated marker resolved (~25 removed) src/coreTypes.ts - GraphVerb: dropped the "@deprecated Will be replaced by HNSWVerbWithMetadata" note. GraphVerb IS the canonical contract — every public API path speaks it. Removed the `source` and `target` legacy alias fields (renamed `from` / `to` callers years ago; no consumers remain). - StorageAdapter: dropped the "@deprecated Use getNouns() with filter" notes from `getNounsByNounType`, `getVerbsBySource`, `getVerbsByTarget`, `getVerbsByType`. They were never deprecated in spirit — they're useful non-paginated convenience wrappers over the paginated `getNouns()` / `getVerbs()` surface. Refreshed JSDoc to explain the role. src/types/graphTypes.ts - Mirrored the GraphVerb cleanup: dropped `source` + `target` legacy aliases. sourceId + targetId are the canonical fields. src/import/ImportCoordinator.ts - Deleted the entire DeprecatedImportOptions interface block (130 LOC). It was a v3 → v4 migration tool using the `?: never` trick to force compile errors on dropped options. Five major versions in, the forced-error gate is no longer pulling its weight. src/triple/TripleIntelligence.ts - Deleted `TripleIntelligenceEngine = any` alias. No consumers; superseded by `TripleIntelligenceSystem`. src/storage/cow/binaryDataCodec.ts - Deleted `wrapBinaryData()`. The COW dispatch layer in `baseStorage.ts` routes by key-prefix convention; the old guess-by-JSON-parse codec was fragile (compressed bytes can accidentally parse as JSON) and unused. src/storage/baseStorage.ts - Refreshed JSDoc on `convertHNSWVerbToGraphVerb()` — the method is alive and used internally; the deprecation note was stale. src/embeddings/wasm/AssetLoader.ts → DELETED - File was @deprecated since model weights moved into the Candle WASM bundle. No consumers. Removed from `embeddings/wasm/index.ts` exports. src/embeddings/wasm/types.ts - Dropped @deprecated tags on `TokenizerConfig` + `TokenizedInput` — still used by `WordPieceTokenizer` (auxiliary tokenization). Deleted `InferenceConfig` (truly dead). Updated `embeddings/wasm/index.ts` exports. src/utils/metadataIndex.ts - Deleted `getIdsForCriteria()` — pure alias for `getIdsForFilter()`, no consumers. src/interfaces/IIndex.ts - Removed RebuildOptions.lazy (deprecated and unused; lazy mode is auto- selected by available-memory detection). src/hnsw/hnswIndex.ts - Removed `getNouns()` (returned a full Map; deprecated in favor of pagination years ago and no consumers in src/ or tests/). PHASE B — cacheManager dead StorageType branches src/storage/cacheManager.ts - Collapsed the `isRemoteStorage` flag and its 15 dead conditional branches spanning calculateOptimalCacheSize() and calculateOptimalBatchSize(). After dropping cloud adapters in step 7, `coldStorageType` is never S3 or REMOTE_API; the branches were dead. Cache sizing and batch sizing now honor the filesystem-only reality with simpler heuristics. - Collapsed `detectWarmStorageType()` + `detectColdStorageType()` from ~40 LOC of environment-+-availability branching to 2-line returns of `StorageType.FILESYSTEM`. Brainy 8.0 ships filesystem + memory only. NOT YET — Phases C-G in follow-up commits C: storageAutoConfig.ts + zeroConfig + extensibleConfig + sharedConfigManager D: TODO/FIXME sweep across src/ E: skipped tests + the parallel-test race condition F: docs deep clean (BATCHING, augmentations, READMEs) G: browser support drop (the last 2 @deprecated) VERIFICATION - npx tsc --noEmit: clean - npm test: 1408 / 1409 (same pre-existing race-condition outstanding) --- src/brainy.ts | 8 +- src/coreTypes.ts | 46 ++-- src/embeddings/wasm/AssetLoader.ts | 266 ----------------------- src/embeddings/wasm/index.ts | 4 +- src/embeddings/wasm/types.ts | 24 +-- src/graph/graphAdjacencyIndex.ts | 2 - src/hnsw/hnswIndex.ts | 8 - src/import/ImportCoordinator.ts | 44 +--- src/interfaces/IIndex.ts | 10 - src/storage/baseStorage.ts | 13 +- src/storage/cacheManager.ts | 336 ++++++----------------------- src/storage/cow/binaryDataCodec.ts | 35 --- src/triple/TripleIntelligence.ts | 4 - src/types/graphTypes.ts | 8 +- src/utils/metadataIndex.ts | 8 - 15 files changed, 98 insertions(+), 718 deletions(-) delete mode 100644 src/embeddings/wasm/AssetLoader.ts diff --git a/src/brainy.ts b/src/brainy.ts index a1e5b092..c25daecc 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1971,8 +1971,6 @@ export class Brainy implements BrainyInterface { vector: relationVector, sourceId: params.from, targetId: params.to, - source: params.from, - target: params.to, verb: params.type, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), @@ -2013,9 +2011,7 @@ export class Brainy implements BrainyInterface { ...verb, id: reverseId, sourceId: params.to, - targetId: params.from, - source: params.to, - target: params.from + targetId: params.from } // Operation 4: Save reverse verb vector data @@ -2162,8 +2158,6 @@ export class Brainy implements BrainyInterface { vector: existingAny.vector, sourceId: existingAny.sourceId, targetId: existingAny.targetId, - source: existingAny.sourceId, - target: existingAny.targetId, verb: newVerbType, type: newVerbType, ...(params.subtype !== undefined diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 5d55ec1f..2f6feb40 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -415,31 +415,25 @@ export interface HNSWVerbWithMetadata { /** * Verb representing a relationship between nouns - * Stored separately from HNSW index for lightweight performance - * - * @deprecated Will be replaced by HNSWVerbWithMetadata in future versions + * Stored separately from the HNSW vector index for lightweight performance. + * This is the canonical verb shape — Brainy's internal graph engine, the public + * `relate()` / `getRelations()` surface, and storage adapters all speak it. */ export interface GraphVerb { id: string // Unique identifier for the verb sourceId: string // ID of the source noun targetId: string // ID of the target noun vector: Vector // Vector representation of the relationship - connections?: Map> // Optional connections from HNSW index - type?: string // Optional type of the relationship + connections?: Map> // Optional connections from the vector index + type?: string // Optional verb type subtype?: string // Optional sub-classification within the VerbType (7.30+) weight?: number // Optional weight of the relationship confidence?: number // Optional confidence score (0-1) metadata?: any // Optional metadata for the verb service?: string // Multi-tenancy support - which service created this verb - - // Legacy field names (use from/to in public API, sourceId/targetId in storage) - /** @deprecated Use `from` (public API) or `sourceId` (storage). Will be removed in next major. */ - source?: string // Entity UUID (same as sourceId) - /** @deprecated Use `to` (public API) or `targetId` (storage). Will be removed in next major. */ - target?: string // Entity UUID (same as targetId) - verb?: string // Alias for type + verb?: string // Alias for type — preserved for ergonomic call sites data?: Record // Additional flexible data storage - embedding?: Vector // Alias for vector + embedding?: Vector // Alias for vector — preserved for ergonomic call sites // Timestamp and creator properties createdAt?: number | { seconds: number; nanoseconds: number } // When the verb was created @@ -800,10 +794,11 @@ export interface StorageAdapter { }> /** - * Get nouns by noun type - * @param nounType The noun type to filter by - * @returns Promise that resolves to an array of nouns of the specified noun type - * @deprecated Use getNouns() with filter.nounType instead + * Get all nouns of a given type. Convenience wrapper over `getNouns()` for + * code paths that need the full set rather than a paginated cursor walk. + * + * @param nounType The noun type to filter by. + * @returns All matching nouns. */ getNounsByNounType(nounType: string): Promise @@ -848,26 +843,19 @@ export interface StorageAdapter { }> /** - * Get verbs by source - * @param sourceId The source ID to filter by - * @returns Promise that resolves to an array of verbs with the specified source ID - * @deprecated Use getVerbs() with filter.sourceId instead + * Get all verbs originating from a given source node. Convenience wrapper + * over `getVerbs()` for code paths that want the full set rather than a + * paginated cursor walk. */ getVerbsBySource(sourceId: string): Promise /** - * Get verbs by target - * @param targetId The target ID to filter by - * @returns Promise that resolves to an array of verbs with the specified target ID - * @deprecated Use getVerbs() with filter.targetId instead + * Get all verbs targeting a given node. See `getVerbsBySource()`. */ getVerbsByTarget(targetId: string): Promise /** - * Get verbs by type - * @param type The verb type to filter by - * @returns Promise that resolves to an array of verbs with the specified type - * @deprecated Use getVerbs() with filter.verbType instead + * Get all verbs of a given type. See `getVerbsBySource()`. */ getVerbsByType(type: string): Promise diff --git a/src/embeddings/wasm/AssetLoader.ts b/src/embeddings/wasm/AssetLoader.ts deleted file mode 100644 index 32fd0d93..00000000 --- a/src/embeddings/wasm/AssetLoader.ts +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Asset Loader - * - * @deprecated This class is no longer used. Model weights are now embedded - * in the Candle WASM binary at compile time. Kept for backward compatibility. - * - * Previously: Resolved paths to ONNX model files across environments. - * Now: Use CandleEmbeddingEngine which loads embedded model automatically. - */ - -import { MODEL_CONSTANTS } from './types.js' - -// Cache resolved paths -let cachedModelDir: string | null = null -let cachedVocab: Record | null = null - -/** - * Asset loader for model files - */ -export class AssetLoader { - private modelDir: string | null = null - - /** - * Get the model directory path - */ - async getModelDir(): Promise { - if (this.modelDir) { - return this.modelDir - } - - if (cachedModelDir) { - this.modelDir = cachedModelDir - return cachedModelDir - } - - // Try to resolve model directory - const resolved = await this.resolveModelDir() - this.modelDir = resolved - cachedModelDir = resolved - return resolved - } - - /** - * Resolve the model directory across environments - */ - private async resolveModelDir(): Promise { - // 1. Check environment variable - if (typeof process !== 'undefined' && process.env?.BRAINY_MODEL_PATH) { - const envPath = process.env.BRAINY_MODEL_PATH - if (await this.pathExists(envPath)) { - return envPath - } - } - - // 2. Try common locations - const modelName = MODEL_CONSTANTS.MODEL_NAME + '-q8' - const possiblePaths = [ - // Package assets (when installed as dependency) - `./assets/models/${modelName}`, - `./node_modules/@soulcraft/brainy/assets/models/${modelName}`, - // Development paths - `../assets/models/${modelName}`, - // Absolute from package root - this.getPackageRootPath(`assets/models/${modelName}`), - ].filter(Boolean) as string[] - - for (const path of possiblePaths) { - if (await this.pathExists(path)) { - return path - } - } - - // If no path found, return default (will error on use) - return `./assets/models/${modelName}` - } - - /** - * Get package root path (Node.js/Bun only) - */ - private getPackageRootPath(relativePath: string): string | null { - if (typeof process === 'undefined') { - return null - } - - try { - // Use __dirname equivalent - const url = new URL(import.meta.url) - const currentDir = url.pathname.replace(/\/[^/]*$/, '') - // Go up from src/embeddings/wasm to package root - const packageRoot = currentDir.replace(/\/src\/embeddings\/wasm$/, '') - return `${packageRoot}/${relativePath}` - } catch { - return null - } - } - - /** - * Check if path exists (works in Node.js/Bun) - */ - private async pathExists(path: string): Promise { - if (typeof process === 'undefined') { - // Browser - check via fetch - try { - const response = await fetch(path, { method: 'HEAD' }) - return response.ok - } catch { - return false - } - } - - // Node.js/Bun - try { - const fs = await import('node:fs/promises') - await fs.access(path) - return true - } catch { - return false - } - } - - /** - * Get path to ONNX model file - */ - async getModelPath(): Promise { - const dir = await this.getModelDir() - return `${dir}/model.onnx` - } - - /** - * Get path to vocabulary file - */ - async getVocabPath(): Promise { - const dir = await this.getModelDir() - return `${dir}/vocab.json` - } - - /** - * Load vocabulary from JSON file - */ - async loadVocab(): Promise> { - if (cachedVocab) { - return cachedVocab - } - - const vocabPath = await this.getVocabPath() - - if (typeof process !== 'undefined') { - // Node.js/Bun - read from filesystem - try { - const fs = await import('node:fs/promises') - const content = await fs.readFile(vocabPath, 'utf-8') - cachedVocab = JSON.parse(content) - return cachedVocab! - } catch (error) { - throw new Error( - `Failed to load vocabulary from ${vocabPath}: ${error instanceof Error ? error.message : String(error)}` - ) - } - } else { - // Browser - fetch - try { - const response = await fetch(vocabPath) - if (!response.ok) { - throw new Error(`HTTP ${response.status}`) - } - cachedVocab = await response.json() - return cachedVocab! - } catch (error) { - throw new Error( - `Failed to fetch vocabulary from ${vocabPath}: ${error instanceof Error ? error.message : String(error)}` - ) - } - } - } - - /** - * Load model as ArrayBuffer (for ONNX session) - */ - async loadModel(): Promise { - const modelPath = await this.getModelPath() - - if (typeof process !== 'undefined') { - // Node.js/Bun - read from filesystem - try { - const fs = await import('node:fs/promises') - const buffer = await fs.readFile(modelPath) - // Convert Node.js Buffer to ArrayBuffer - return new Uint8Array(buffer).buffer as ArrayBuffer - } catch (error) { - throw new Error( - `Failed to load model from ${modelPath}: ${error instanceof Error ? error.message : String(error)}` - ) - } - } else { - // Browser - fetch - try { - const response = await fetch(modelPath) - if (!response.ok) { - throw new Error(`HTTP ${response.status}`) - } - return await response.arrayBuffer() - } catch (error) { - throw new Error( - `Failed to fetch model from ${modelPath}: ${error instanceof Error ? error.message : String(error)}` - ) - } - } - } - - /** - * Verify all required assets exist - */ - async verifyAssets(): Promise<{ - valid: boolean - modelPath: string - vocabPath: string - errors: string[] - }> { - const errors: string[] = [] - const modelPath = await this.getModelPath() - const vocabPath = await this.getVocabPath() - - if (!(await this.pathExists(modelPath))) { - errors.push(`Model file not found: ${modelPath}`) - } - - if (!(await this.pathExists(vocabPath))) { - errors.push(`Vocabulary file not found: ${vocabPath}`) - } - - return { - valid: errors.length === 0, - modelPath, - vocabPath, - errors, - } - } - - /** - * Clear cached paths (for testing) - */ - clearCache(): void { - this.modelDir = null - cachedModelDir = null - cachedVocab = null - } -} - -/** - * Create asset loader instance - */ -export function createAssetLoader(): AssetLoader { - return new AssetLoader() -} - -/** - * Singleton asset loader - */ -let singletonLoader: AssetLoader | null = null - -export function getAssetLoader(): AssetLoader { - if (!singletonLoader) { - singletonLoader = new AssetLoader() - } - return singletonLoader -} diff --git a/src/embeddings/wasm/index.ts b/src/embeddings/wasm/index.ts index 014f5bb5..69886443 100644 --- a/src/embeddings/wasm/index.ts +++ b/src/embeddings/wasm/index.ts @@ -25,16 +25,14 @@ export { cosineSimilarity, } from './CandleEmbeddingEngine.js' -// Legacy components (for backward compatibility - not needed with Candle) +// Auxiliary embedding components export { WordPieceTokenizer, createTokenizer } from './WordPieceTokenizer.js' export { EmbeddingPostProcessor, createPostProcessor } from './EmbeddingPostProcessor.js' -export { AssetLoader, getAssetLoader, createAssetLoader } from './AssetLoader.js' // Types export type { TokenizerConfig, TokenizedInput, - InferenceConfig, EmbeddingResult, EngineStats, ModelConfig, diff --git a/src/embeddings/wasm/types.ts b/src/embeddings/wasm/types.ts index f59b2f87..551fe26e 100644 --- a/src/embeddings/wasm/types.ts +++ b/src/embeddings/wasm/types.ts @@ -6,8 +6,8 @@ */ /** - * Tokenizer configuration for WordPiece - * @deprecated Tokenization now happens in Rust/WASM - kept for backward compatibility + * Tokenizer configuration for WordPiece. Used by `WordPieceTokenizer` for the + * JS-side tokenization path (auxiliary to the Rust/WASM embedding engine). */ export interface TokenizerConfig { /** Vocabulary mapping word → token ID */ @@ -27,8 +27,7 @@ export interface TokenizerConfig { } /** - * Result of tokenization - * @deprecated Tokenization now happens in Rust/WASM - kept for backward compatibility + * Result of tokenization. Returned by `WordPieceTokenizer.encode()`. */ export interface TokenizedInput { /** Token IDs including [CLS] and [SEP] */ @@ -41,23 +40,6 @@ export interface TokenizedInput { tokenCount: number } -/** - * Inference engine configuration - * @deprecated Model is now embedded in WASM - kept for backward compatibility - */ -export interface InferenceConfig { - /** Path to model file (not used with embedded model) */ - modelPath: string - /** Path to WASM files directory */ - wasmPath?: string - /** Number of threads (1 for universal compatibility) */ - numThreads: number - /** Enable SIMD if available */ - enableSimd: boolean - /** Enable CPU memory arena */ - enableCpuMemArena: boolean -} - /** * Embedding result with metadata */ diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index ddca800f..a3a9faea 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -645,8 +645,6 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { sourceId: verb.sourceId, targetId: verb.targetId, vector: verb.vector, - source: verb.sourceId, - target: verb.targetId, verb: verb.verb, createdAt: { seconds: Math.floor(verb.createdAt / 1000), nanoseconds: (verb.createdAt % 1000) * 1000000 }, updatedAt: { seconds: Math.floor(verb.updatedAt / 1000), nanoseconds: (verb.updatedAt % 1000) * 1000000 }, diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 06806658..a6a2ce04 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -1063,14 +1063,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return true } - /** - * Get all nouns in the index - * @deprecated Use getNounsPaginated() instead for better scalability - */ - public getNouns(): Map { - return new Map(this.nouns) - } - /** * Get nouns with pagination * @param options Pagination options diff --git a/src/import/ImportCoordinator.ts b/src/import/ImportCoordinator.ts index 5d480050..f53dc120 100644 --- a/src/import/ImportCoordinator.ts +++ b/src/import/ImportCoordinator.ts @@ -198,49 +198,9 @@ export interface ValidImportOptions { } /** - * Deprecated import options from v3.x - * Using these will cause TypeScript compile errors - * - * @deprecated These options are no longer supported in v4.x - * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} + * Complete import options interface. */ -export interface DeprecatedImportOptions { - /** - * @deprecated Use `enableRelationshipInference` instead - * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} - */ - extractRelationships?: never - - /** - * @deprecated Removed in v4.x - auto-detection is now always enabled - * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} - */ - autoDetect?: never - - /** - * @deprecated Use `vfsPath` to specify the directory path instead - * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} - */ - createFileStructure?: never - - /** - * @deprecated Removed in v4.x - all sheets are now processed automatically - * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} - */ - excelSheets?: never - - /** - * @deprecated Removed in v4.x - table extraction is now automatic for PDF imports - * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} - */ - pdfExtractTables?: never -} - -/** - * Complete import options interface - * Combines valid v4.x options with deprecated v3.x options (which cause TypeScript errors) - */ -export type ImportOptions = ValidImportOptions & DeprecatedImportOptions +export type ImportOptions = ValidImportOptions export interface ImportProgress { stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete' diff --git a/src/interfaces/IIndex.ts b/src/interfaces/IIndex.ts index b88ca8dc..4be80567 100644 --- a/src/interfaces/IIndex.ts +++ b/src/interfaces/IIndex.ts @@ -50,16 +50,6 @@ export type RebuildProgressCallback = (loaded: number, total: number) => void * Rebuild options for index rebuilding */ export interface RebuildOptions { - /** - * @deprecated Lazy mode is now auto-detected based on available memory. - * System automatically chooses between: - * - Preloading: Small datasets that fit comfortably in cache (< 80% threshold) - * - On-demand: Large datasets loaded adaptively via UnifiedCache - * - * This option is kept for backwards compatibility but is ignored. - * The system always uses adaptive caching. - */ - lazy?: boolean /** * Batch size for pagination during rebuild diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index c600038c..b26c661c 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -28,7 +28,7 @@ import { getShardIdFromUuid } from './sharding.js' import { RefManager } from './cow/RefManager.js' import { BlobStorage, type COWStorageAdapter } from './cow/BlobStorage.js' import { CommitLog } from './cow/CommitLog.js' -import { unwrapBinaryData, wrapBinaryData } from './cow/binaryDataCodec.js' +import { unwrapBinaryData } from './cow/binaryDataCodec.js' import { prodLog } from '../utils/logger.js' import { MetadataWriteBuffer } from '../utils/metadataWriteBuffer.js' @@ -1126,10 +1126,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { } /** - * Convert HNSWVerb to GraphVerb by combining with metadata - * DEPRECATED: For backward compatibility only. Use getVerb() which returns HNSWVerbWithMetadata. - * - * @deprecated Use getVerb() instead which returns HNSWVerbWithMetadata + * Convert an HNSW verb (raw vector store entry) into a `GraphVerb` shape by + * combining with the verb's metadata. Used by internal code paths that need + * the graph-layer shape rather than the storage-layer shape. */ protected async convertHNSWVerbToGraphVerb(hnswVerb: HNSWVerb): Promise { try { @@ -1169,10 +1168,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { sourceId: hnswVerb.sourceId, targetId: hnswVerb.targetId, - // Aliases for backward compatibility + // Alias for ergonomic access type: hnswVerb.verb, - source: hnswVerb.sourceId, - target: hnswVerb.targetId, // Optional fields from metadata file weight: metadata?.weight || 1.0, diff --git a/src/storage/cacheManager.ts b/src/storage/cacheManager.ts index 56c1af2e..5575f079 100644 --- a/src/storage/cacheManager.ts +++ b/src/storage/cacheManager.ts @@ -623,118 +623,54 @@ export class CacheManager { private async tuneHotCacheSize(): Promise { // Use the async version to get more accurate memory information let optimalSize = await this.detectOptimalCacheSizeAsync() - - // Check if we're in read-only mode + const isReadOnly = this.options?.readOnly || false - - // Check if we're using S3 or other remote storage - const isRemoteStorage = - this.coldStorageType === StorageType.S3 || - this.coldStorageType === StorageType.REMOTE_API - - // If we have storage statistics, adjust based on total nodes/edges + + // Adjust by total entity count if we have storage statistics. if (this.storageStatistics) { - const totalItems = (this.storageStatistics.totalNodes || 0) + + const totalItems = (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) - - // If total items is significant, adjust cache size + if (totalItems > 0) { - // Base percentage to cache - adjusted based on mode and storage - let percentageToCache = 0.2 // Cache 20% of items by default - - // For read-only mode, increase cache percentage - if (isReadOnly) { - percentageToCache = 0.3 // 30% for read-only mode - - // For remote storage in read-only mode, be even more aggressive - if (isRemoteStorage) { - percentageToCache = 0.4 // 40% for remote storage in read-only mode - } - } - // For remote storage in normal mode, increase slightly - else if (isRemoteStorage) { - percentageToCache = 0.25 // 25% for remote storage - } - - // For large datasets, cap the percentage to avoid excessive memory usage - if (totalItems > 1000000) { // Over 1 million items + let percentageToCache = isReadOnly ? 0.3 : 0.2 + + if (totalItems > 1_000_000) { percentageToCache = Math.min(percentageToCache, 0.15) - } else if (totalItems > 100000) { // Over 100K items + } else if (totalItems > 100_000) { percentageToCache = Math.min(percentageToCache, 0.25) } - + const statisticsBasedSize = Math.ceil(totalItems * percentageToCache) - - // Use the smaller of the two to avoid memory issues optimalSize = Math.min(optimalSize, statisticsBasedSize) } } - - // Adjust based on hit/miss ratio if we have enough data + + // Adjust by observed hit ratio. const totalAccesses = this.stats.hits + this.stats.misses if (totalAccesses > 100) { const hitRatio = this.stats.hits / totalAccesses - - // Base adjustment factor - let hitRatioFactor = 1.0 - - // If hit ratio is low, we might need a larger cache + if (hitRatio < 0.5) { - // Calculate adjustment factor based on hit ratio const baseAdjustment = 0.5 - hitRatio - - // For read-only mode or remote storage, be more aggressive - if (isReadOnly || isRemoteStorage) { - hitRatioFactor = 1 + (baseAdjustment * 1.5) // Up to 75% increase - } else { - hitRatioFactor = 1 + baseAdjustment // Up to 50% increase - } - - optimalSize = Math.ceil(optimalSize * hitRatioFactor) - } - // If hit ratio is very high, we might be able to reduce cache size slightly - else if (hitRatio > 0.9 && !isReadOnly && !isRemoteStorage) { - // Only reduce cache size in normal mode with local storage - // and only if hit ratio is very high - hitRatioFactor = 0.9 // 10% reduction + const hitRatioFactor = isReadOnly ? 1 + baseAdjustment * 1.5 : 1 + baseAdjustment optimalSize = Math.ceil(optimalSize * hitRatioFactor) + } else if (hitRatio > 0.9 && !isReadOnly) { + optimalSize = Math.ceil(optimalSize * 0.9) } } - - // Check for operation patterns if available + + // Read-heavy workloads warrant a slightly larger cache. if (this.storageStatistics?.operations) { const ops = this.storageStatistics.operations const totalOps = ops.total || 1 - - // Calculate read/write ratio const readOps = (ops.search || 0) + (ops.get || 0) - const writeOps = (ops.add || 0) + (ops.update || 0) + (ops.delete || 0) - - if (totalOps > 100) { - const readRatio = readOps / totalOps - - // For read-heavy workloads, increase cache size - if (readRatio > 0.8) { - // More aggressive for remote storage - const readAdjustment = isRemoteStorage ? 1.3 : 1.2 - optimalSize = Math.ceil(optimalSize * readAdjustment) - } + + if (totalOps > 100 && readOps / totalOps > 0.8) { + optimalSize = Math.ceil(optimalSize * 1.2) } } - - // Ensure we have a reasonable minimum size based on environment and mode - let minSize = 1000 // Default minimum - - // For read-only mode, use a higher minimum - if (isReadOnly) { - minSize = 2000 - } - - // For remote storage, use an even higher minimum - if (isRemoteStorage) { - minSize = isReadOnly ? 3000 : 2000 - } - + + const minSize = isReadOnly ? 2000 : 1000 optimalSize = Math.max(optimalSize, minSize) // Update the hot cache max size @@ -941,98 +877,36 @@ export class CacheManager { * @param cacheStats Optional cache statistics for more adaptive tuning */ private tuneBatchSize(cacheStats?: CacheStats): void { - // Default batch size let batchSize = 10 - - // Use provided cache stats or internal stats const stats = cacheStats || this.getStats() - - // Check if we're in read-only mode const isReadOnly = this.options?.readOnly || false - - // Check if we're using S3 or other remote storage - const isRemoteStorage = - this.coldStorageType === StorageType.S3 || - this.coldStorageType === StorageType.REMOTE_API - - // Get the total dataset size if available - const totalItems = this.storageStatistics ? - (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0 - - // Determine if we're dealing with a large dataset - const isLargeDataset = totalItems > 100000 - const isVeryLargeDataset = totalItems > 1000000 - - // Base batch size adjustment based on environment - if (this.environment === Environment.NODE) { - // Node.js can handle larger batches - batchSize = isReadOnly ? 30 : 20 - - // For remote storage, increase batch size - if (isRemoteStorage) { - batchSize = isReadOnly ? 50 : 30 - } - - // For large datasets, adjust batch size - if (isLargeDataset) { - batchSize = Math.min(100, batchSize * 1.5) - } - - // For very large datasets, adjust even more - if (isVeryLargeDataset) { - batchSize = Math.min(200, batchSize * 2) - } - } else if (this.environment === Environment.BROWSER) { - // Browsers might need smaller batches - batchSize = isReadOnly ? 15 : 10 - - // If we have memory information, adjust accordingly - if (navigator.deviceMemory) { - // Scale batch size with available memory - const memoryFactor = isReadOnly ? 3 : 2 - batchSize = Math.max(5, Math.min(30, Math.floor(navigator.deviceMemory * memoryFactor))) - - // For large datasets, adjust based on memory - if (isLargeDataset && navigator.deviceMemory > 4) { - batchSize = Math.min(50, batchSize * 1.5) - } - } - } else if (this.environment === Environment.WORKER) { - // Workers can handle moderate batch sizes - batchSize = isReadOnly ? 20 : 15 - } - - // Adjust based on cache hit/miss ratios + + const totalItems = this.storageStatistics + ? (this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) + : 0 + const isLargeDataset = totalItems > 100_000 + const isVeryLargeDataset = totalItems > 1_000_000 + + // Brainy 8.0 ships Node-like runtimes only (Bun, Deno, Node). All paths + // funnel into the same batch-size envelope. + batchSize = isReadOnly ? 30 : 20 + if (isLargeDataset) batchSize = Math.min(100, batchSize * 1.5) + if (isVeryLargeDataset) batchSize = Math.min(200, batchSize * 2) + + // Adjust by hit-ratio observations. const totalHotAccesses = stats.hotCacheHits + stats.hotCacheMisses const totalWarmAccesses = stats.warmCacheHits + stats.warmCacheMisses - + if (totalHotAccesses > 100) { const hotHitRatio = stats.hotCacheHits / totalHotAccesses - - // If hot cache hit ratio is high, we're effectively using the cache - // so we can use larger batches for better throughput - if (hotHitRatio > 0.8) { - // High hit ratio, increase batch size - batchSize = Math.min(batchSize * 1.5, isRemoteStorage ? 250 : 150) - } else if (hotHitRatio < 0.4) { - // Low hit ratio, we might be fetching too much at once - // Reduce batch size to be more selective - batchSize = Math.max(5, batchSize * 0.8) - } + if (hotHitRatio > 0.8) batchSize = Math.min(batchSize * 1.5, 150) + else if (hotHitRatio < 0.4) batchSize = Math.max(5, batchSize * 0.8) } - + if (totalWarmAccesses > 50) { const warmHitRatio = stats.warmCacheHits / totalWarmAccesses - - // If warm cache hit ratio is high, prefetching is effective - // so we can use larger batches - if (warmHitRatio > 0.7) { - // High warm hit ratio, increase batch size - batchSize = Math.min(batchSize * 1.3, isRemoteStorage ? 200 : 120) - } else if (warmHitRatio < 0.3) { - // Low warm hit ratio, reduce batch size - batchSize = Math.max(5, batchSize * 0.9) - } + if (warmHitRatio > 0.7) batchSize = Math.min(batchSize * 1.3, 120) + else if (warmHitRatio < 0.3) batchSize = Math.max(5, batchSize * 0.9) } // If we have storage statistics with operation counts, adjust based on operation patterns @@ -1043,140 +917,64 @@ export class CacheManager { const getOps = (ops.get || 0) if (totalOps > 100) { - // Calculate search and get ratios const searchRatio = searchOps / totalOps const getRatio = getOps / totalOps - - // For search-heavy workloads, use larger batch size + if (searchRatio > 0.6) { - // Search-heavy, increase batch size - const searchFactor = isRemoteStorage ? 1.8 : 1.5 - batchSize = Math.min(isRemoteStorage ? 200 : 100, Math.ceil(batchSize * searchFactor)) + batchSize = Math.min(100, Math.ceil(batchSize * 1.5)) } - - // For get-heavy workloads, adjust batch size if (getRatio > 0.6) { - // Get-heavy, adjust batch size based on storage type - if (isRemoteStorage) { - // For remote storage, larger batches reduce network overhead - batchSize = Math.min(150, Math.ceil(batchSize * 1.5)) - } else { - // For local storage, smaller batches might be more efficient - batchSize = Math.max(10, Math.ceil(batchSize * 0.9)) - } + batchSize = Math.max(10, Math.ceil(batchSize * 0.9)) } } } - - // Check if we're experiencing memory pressure + + // Memory-pressure trim. if (stats.hotCacheSize > 0 && this.hotCacheMaxSize > 0) { const cacheUtilization = stats.hotCacheSize / this.hotCacheMaxSize - - // If cache utilization is high, reduce batch size to avoid memory pressure if (cacheUtilization > 0.85) { batchSize = Math.max(5, Math.floor(batchSize * 0.8)) } } - - // Adjust based on overall hit/miss ratio if we have enough data + + // Combined hot+warm hit-ratio adjustment. const totalAccesses = stats.hotCacheHits + stats.hotCacheMisses + stats.warmCacheHits + stats.warmCacheMisses if (totalAccesses > 100) { const hitRatio = (stats.hotCacheHits + stats.warmCacheHits) / totalAccesses - - // Base adjustment factors - let increaseFactorForLowHitRatio = isRemoteStorage ? 1.5 : 1.2 - let decreaseFactorForHighHitRatio = 0.8 - - // In read-only mode, be more aggressive with batch size adjustments - if (isReadOnly) { - increaseFactorForLowHitRatio = isRemoteStorage ? 2.0 : 1.5 - decreaseFactorForHighHitRatio = 0.9 // Less reduction in read-only mode - } - - // If hit ratio is high, we can use smaller batches + const increaseFactorForLowHitRatio = isReadOnly ? 1.5 : 1.2 + const decreaseFactorForHighHitRatio = isReadOnly ? 0.9 : 0.8 + if (hitRatio > 0.8 && !isVeryLargeDataset) { - // High hit ratio, decrease batch size slightly - // But don't decrease too much for large datasets or remote storage - if (!(isLargeDataset && isRemoteStorage)) { - batchSize = Math.max(isReadOnly ? 10 : 5, Math.floor(batchSize * decreaseFactorForHighHitRatio)) - } - } - // If hit ratio is low, we need larger batches - else if (hitRatio < 0.5) { - // Low hit ratio, increase batch size - const maxBatchSize = isRemoteStorage ? - (isVeryLargeDataset ? 300 : 200) : - (isVeryLargeDataset ? 150 : 100) - + batchSize = Math.max(isReadOnly ? 10 : 5, Math.floor(batchSize * decreaseFactorForHighHitRatio)) + } else if (hitRatio < 0.5) { + const maxBatchSize = isVeryLargeDataset ? 150 : 100 batchSize = Math.min(maxBatchSize, Math.ceil(batchSize * increaseFactorForLowHitRatio)) } } - - // Set minimum batch sizes based on storage type and mode - let minBatchSize = 5 - - if (isRemoteStorage) { - minBatchSize = isReadOnly ? 20 : 10 - } else if (isReadOnly) { - minBatchSize = 10 - } - - // Ensure batch size is within reasonable limits + + // Min/max envelope. 8.0 is Node-like only. + const minBatchSize = isReadOnly ? 10 : 5 batchSize = Math.max(minBatchSize, batchSize) - - // Cap maximum batch size based on environment and storage - const maxBatchSize = isRemoteStorage ? - (this.environment === Environment.NODE ? 300 : 150) : - (this.environment === Environment.NODE ? 150 : 75) - - batchSize = Math.min(maxBatchSize, batchSize) + batchSize = Math.min(150, batchSize) // Update the batch size with the adaptively tuned value this.batchSize = Math.round(batchSize) } /** - * Detect the appropriate warm storage type based on environment + * Resolve the warm-tier storage type. Brainy 8.0 ships filesystem + + * memory only. */ private detectWarmStorageType(): StorageType { - if (this.environment === Environment.BROWSER) { - // Use OPFS if available, otherwise use memory - if ('storage' in navigator && 'getDirectory' in navigator.storage) { - return StorageType.OPFS - } - return StorageType.MEMORY - } else if (this.environment === Environment.WORKER) { - // Use OPFS if available, otherwise use memory - if ('storage' in self && 'getDirectory' in (self as WorkerGlobalScope).storage!) { - return StorageType.OPFS - } - return StorageType.MEMORY - } else { - // In Node.js, use filesystem - return StorageType.FILESYSTEM - } + return StorageType.FILESYSTEM } - + /** - * Detect the appropriate cold storage type based on environment + * Resolve the cold-tier storage type. Brainy 8.0 ships filesystem + + * memory only. */ private detectColdStorageType(): StorageType { - if (this.environment === Environment.BROWSER) { - // Use OPFS if available, otherwise use memory - if ('storage' in navigator && 'getDirectory' in navigator.storage) { - return StorageType.OPFS - } - return StorageType.MEMORY - } else if (this.environment === Environment.WORKER) { - // Use OPFS if available, otherwise use memory - if ('storage' in self && 'getDirectory' in (self as WorkerGlobalScope).storage!) { - return StorageType.OPFS - } - return StorageType.MEMORY - } else { - // In Node.js, use S3 if configured, otherwise filesystem - return StorageType.S3 - } + return StorageType.FILESYSTEM } /** diff --git a/src/storage/cow/binaryDataCodec.ts b/src/storage/cow/binaryDataCodec.ts index 68c393dd..296daf2b 100644 --- a/src/storage/cow/binaryDataCodec.ts +++ b/src/storage/cow/binaryDataCodec.ts @@ -83,41 +83,6 @@ export function unwrapBinaryData(data: any): Buffer { ) } -/** - * Wrap binary data for JSON storage - * - * ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! - * ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead. - * ⚠️ This function exists for legacy/compatibility only. - * - * DEPRECATED APPROACH: Tries to guess if data is JSON by parsing. - * This is FRAGILE because compressed binary can accidentally parse as valid JSON, - * causing blob integrity failures. - * - * SOLUTION: baseStorage.ts COW adapter now uses key naming convention: - * - Keys with '-meta:' or 'ref:' prefix → Always JSON - * - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary - * No guessing needed! - * - * @param data - Buffer to wrap - * @returns Wrapped object or parsed JSON object - * @deprecated Use key-based dispatch in baseStorage.ts instead - */ -export function wrapBinaryData(data: Buffer): any { - // Try to parse as JSON first (for metadata, trees, commits) - // NOTE: This is the OLD approach - fragile because compressed data - // can accidentally parse as valid JSON! - try { - return JSON.parse(data.toString()) - } catch { - // Not JSON - wrap as binary data - return { - _binary: true, - data: data.toString('base64') - } as WrappedBinaryData - } -} - /** * Ensure data is a Buffer * diff --git a/src/triple/TripleIntelligence.ts b/src/triple/TripleIntelligence.ts index b4a7c521..18782ac2 100644 --- a/src/triple/TripleIntelligence.ts +++ b/src/triple/TripleIntelligence.ts @@ -61,7 +61,3 @@ export interface QueryPlan { estimatedCost: number } -/** - * @deprecated Use brain.getTripleIntelligence() directly to get TripleIntelligenceSystem - */ -export type TripleIntelligenceEngine = any \ No newline at end of file diff --git a/src/types/graphTypes.ts b/src/types/graphTypes.ts index 2ebf340e..57c2afdb 100644 --- a/src/types/graphTypes.ts +++ b/src/types/graphTypes.ts @@ -369,12 +369,8 @@ export interface GraphNoun { */ export interface GraphVerb { id: string // Unique identifier for the verb - /** @deprecated Use `from` (public API) or `sourceId` (storage). Will be removed in next major. */ - source: string // Entity UUID of the source noun - /** @deprecated Use `to` (public API) or `targetId` (storage). Will be removed in next major. */ - target: string // Entity UUID of the target noun - sourceId?: string // Entity UUID of the source noun (storage convention) - targetId?: string // Entity UUID of the target noun (storage convention) + sourceId: string // Entity UUID of the source noun + targetId: string // Entity UUID of the target noun label?: string // Optional descriptive label verb: VerbType // Type of relationship createdAt: Timestamp | number // When the verb was created diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 9d38bf60..e71975b4 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2309,14 +2309,6 @@ export class MetadataIndexManager implements MetadataIndexProvider { ) } - /** - * Get IDs matching multiple criteria (intersection) - LEGACY METHOD - * @deprecated Use getIdsForFilter instead - */ - async getIdsForCriteria(criteria: Record): Promise { - return this.getIdsForFilter(criteria) - } - /** * Flush dirty entries to storage (non-blocking version) * NOTE: Sparse indices are flushed immediately in add/remove operations