soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

chore(8.0): collapse dead defensive guards + redundant polyfills

Browse source 
Follow-up to the browser/cloud/threading sweep — everything that was guarding
against unreachable runtimes is now dead.

cacheManager.ts: Environment enum + this.environment field removed (only NODE
was reachable). StorageType narrowed to MEMORY + FILESYSTEM. navigator.deviceMemory
and performance.memory paths in detectOptimalCacheSize + detectAvailableMemory
deleted; node:os is the sole source. environmentConfig keeps the index signature
for future per-runtime tuning but only the node slot is wired.

Dead 'typeof window === undefined' guards (the check is always true on 8.0):
paramValidation, structuredLogger, mutex, brainy.ts stats block, and
networkTransport ws-dynamic-import all collapsed. IntegrationLoader's inverse
guard ('!== undefined' returning 'browser') deleted. mutex's createMutex default
type simplified from "(typeof window === 'undefined' ? 'file' : 'memory')" to
plain 'file'.

Redundant TextEncoder/TextDecoder polyfills: Node 22+ ships both as globals.
src/utils/textEncoding.ts deleted (applyTensorFlowPatch was named for a defunct
dep and only re-assigned globals already present). setup.ts collapsed to an
empty stable import target. unified.ts loses its applyTensorFlowPatch re-export.

modelAutoConfig.getModelPath() loses the unreachable trailing fallback now
that isNode() is effectively the only branch the function ever takes.

jsonProcessing.ts left unchanged — its 'typeof document' checks are value-shape
guards on the parsed JSON, not environment detection.
This commit is contained in:
David Snelling 2026-06-09 16:46:16 -07:00
parent 266715aeee
commit 42159f2bd7
11 changed files with 121 additions and 397 deletions
Download patch file Download diff file Expand all files Collapse all files

323
src/storage/cacheManager.ts
View file

@ -10,21 +10,6 @@
import { HNSWNoun, GraphVerb, HNSWVerb } from '../coreTypes.js'
import { BrainyError } from '../errors/brainyError.js'
// Extend Navigator interface to include deviceMemory property
// and WorkerGlobalScope to include storage property
declare global {
interface Navigator {
deviceMemory?: number;
}
interface WorkerGlobalScope {
storage?: {
getDirectory?: () => Promise<any>;
[key: string]: any;
};
}
}
// Type aliases for better readability
type HNSWNode = HNSWNoun
type Edge = GraphVerb
@ -52,20 +37,11 @@ interface CacheStats {
warmCacheMisses: number
}
// Environment detection for storage selection
enum Environment {
BROWSER,
NODE,
WORKER
}
// Storage type for warm and cold caches
// Storage type for warm and cold caches. Brainy 8.0 only ships filesystem +
// memory tiers, but the enum is retained for diagnostic/log output.
enum StorageType {
MEMORY,
OPFS,
FILESYSTEM,
S3,
REMOTE_API
FILESYSTEM
}
/**
@ -90,8 +66,7 @@ export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
warmCacheMisses: 0
}
// Environment and storage configuration
private environment: Environment
// Storage configuration
private warmStorageType: StorageType
private coldStorageType: StorageType
@ -127,19 +102,7 @@ export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
hotCacheEvictionThreshold?: number
warmCacheTTL?: number
batchSize?: number
},
browser?: {
hotCacheMaxSize?: number
hotCacheEvictionThreshold?: number
warmCacheTTL?: number
batchSize?: number
},
worker?: {
hotCacheMaxSize?: number
hotCacheEvictionThreshold?: number
warmCacheTTL?: number
batchSize?: number
},
}
[key: string]: {
hotCacheMaxSize?: number
hotCacheEvictionThreshold?: number
@ -148,7 +111,7 @@ export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
} | undefined
}
}
/**
* Initialize the cache manager
* @param options Configuration options
@ -168,19 +131,7 @@ export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
hotCacheEvictionThreshold?: number
warmCacheTTL?: number
batchSize?: number
},
browser?: {
hotCacheMaxSize?: number
hotCacheEvictionThreshold?: number
warmCacheTTL?: number
batchSize?: number
},
worker?: {
hotCacheMaxSize?: number
hotCacheEvictionThreshold?: number
warmCacheTTL?: number
batchSize?: number
},
}
[key: string]: {
hotCacheMaxSize?: number
hotCacheEvictionThreshold?: number
@ -191,39 +142,37 @@ export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
} = {}) {
// Store options for later reference
this.options = options
// Detect environment
this.environment = this.detectEnvironment()
// Set storage types based on environment
// Set storage types (Brainy 8.0 ships filesystem only)
this.warmStorageType = this.detectWarmStorageType()
this.coldStorageType = this.detectColdStorageType()
// Initialize storage adapters
this.warmStorage = options.warmStorage || this.initializeWarmStorage()
this.coldStorage = options.coldStorage || this.initializeColdStorage()
// Set auto-tuning flag
this.autoTune = options.autoTune !== undefined ? options.autoTune : true
// Get environment-specific configuration if available
const envConfig = options.environmentConfig?.[Environment[this.environment].toLowerCase()]
// Brainy 8.0 only runs on Node-like runtimes, so only the `node` slot of
// environmentConfig is honored.
const envConfig = options.environmentConfig?.node
// Set default values or use environment-specific values or global values
this.hotCacheMaxSize = envConfig?.hotCacheMaxSize || options.hotCacheMaxSize || this.detectOptimalCacheSize()
this.hotCacheEvictionThreshold = envConfig?.hotCacheEvictionThreshold || options.hotCacheEvictionThreshold || 0.8
this.warmCacheTTL = envConfig?.warmCacheTTL || options.warmCacheTTL || 24 * 60 * 60 * 1000 // 24 hours
this.batchSize = envConfig?.batchSize || options.batchSize || 10
// If auto-tuning is enabled, perform initial tuning
if (this.autoTune) {
this.tuneParameters()
}
// Log configuration
if (process.env.DEBUG) {
console.log('Cache Manager initialized with configuration:', {
environment: Environment[this.environment],
environment: 'node',
hotCacheMaxSize: this.hotCacheMaxSize,
hotCacheEvictionThreshold: this.hotCacheEvictionThreshold,
warmCacheTTL: this.warmCacheTTL,
@ -234,21 +183,8 @@ export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
})
}
}
/**
* Detect the current environment
*/
private detectEnvironment(): Environment {
if (typeof window !== 'undefined' && typeof document !== 'undefined') {
return Environment.BROWSER
} else if (typeof self !== 'undefined' && typeof window === 'undefined') {
// In a worker environment, self is defined but window is not
return Environment.WORKER
} else {
return Environment.NODE
}
}
/**
* Detect the optimal cache size based on available memory and operating mode
*
@ -262,111 +198,65 @@ export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
try {
// Default to a conservative value
const defaultSize = 1000
// Get the total dataset size if available
const totalItems = this.storageStatistics ?
const totalItems = this.storageStatistics ?
(this.storageStatistics.totalNodes || 0) + (this.storageStatistics.totalEdges || 0) : 0
// Determine if we're dealing with a large dataset (>100K items)
const isLargeDataset = totalItems > 100000
// Check if we're in read-only mode (from parent Brainy instance)
const isReadOnly = this.options?.readOnly || false
// In Node.js, use available system memory with enhanced allocation
if (this.environment === Environment.NODE) {
try {
// For ES module compatibility, we'll use a fixed default value
// since we can't use dynamic imports in a synchronous function
// Use conservative defaults that don't require OS module
// These values are reasonable for most systems
const estimatedTotalMemory = 8 * 1024 * 1024 * 1024 // Assume 8GB total
const estimatedFreeMemory = 4 * 1024 * 1024 * 1024 // Assume 4GB free
// Estimate average entry size (in bytes)
// This is a conservative estimate for complex objects with vectors
const ESTIMATED_BYTES_PER_ENTRY = 1024 // 1KB per entry
// Base memory percentage - 10% by default
let memoryPercentage = 0.1
// Adjust based on operating mode and dataset size
if (isReadOnly) {
// In read-only mode, we can use more memory for caching
memoryPercentage = 0.25 // 25% of free memory
// For large datasets in read-only mode, be even more aggressive
if (isLargeDataset) {
memoryPercentage = 0.4 // 40% of free memory
}
} else if (isLargeDataset) {
// For large datasets in normal mode, increase slightly
memoryPercentage = 0.15 // 15% of free memory
}
// Calculate optimal size based on adjusted percentage
const optimalSize = Math.max(
Math.floor(estimatedFreeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY),
1000
)
// If we know the total dataset size, cap at a reasonable percentage
if (totalItems > 0) {
// In read-only mode, we can cache a larger percentage
const maxPercentage = isReadOnly ? 0.5 : 0.3
const maxItems = Math.ceil(totalItems * maxPercentage)
// Return the smaller of the two to avoid excessive memory usage
return Math.min(optimalSize, maxItems)
}
return optimalSize
} catch (error) {
console.warn('Failed to detect optimal cache size:', error)
return defaultSize
}
}
// In browser, use navigator.deviceMemory with enhanced allocation
if (this.environment === Environment.BROWSER && navigator.deviceMemory) {
// Base entries per GB
let entriesPerGB = 500
try {
// Synchronous path can't use dynamic imports, so we use conservative
// assumed defaults. The async variant (detectAvailableMemory) reads
// real values via `node:os` when possible.
const estimatedFreeMemory = 4 * 1024 * 1024 * 1024 // Assume 4GB free
// Estimate average entry size (in bytes)
// This is a conservative estimate for complex objects with vectors
const ESTIMATED_BYTES_PER_ENTRY = 1024 // 1KB per entry
// Base memory percentage - 10% by default
let memoryPercentage = 0.1
// Adjust based on operating mode and dataset size
if (isReadOnly) {
entriesPerGB = 800 // More aggressive caching in read-only mode
// In read-only mode, we can use more memory for caching
memoryPercentage = 0.25 // 25% of free memory
// For large datasets in read-only mode, be even more aggressive
if (isLargeDataset) {
entriesPerGB = 1000 // Even more aggressive for large datasets
memoryPercentage = 0.4 // 40% of free memory
}
} else if (isLargeDataset) {
entriesPerGB = 600 // Slightly more aggressive for large datasets
// For large datasets in normal mode, increase slightly
memoryPercentage = 0.15 // 15% of free memory
}
// Calculate based on device memory
const browserCacheSize = Math.max(navigator.deviceMemory * entriesPerGB, 1000)
// Calculate optimal size based on adjusted percentage
const optimalSize = Math.max(
Math.floor(estimatedFreeMemory * memoryPercentage / ESTIMATED_BYTES_PER_ENTRY),
1000
)
// If we know the total dataset size, cap at a reasonable percentage
if (totalItems > 0) {
// In read-only mode, we can cache a larger percentage
const maxPercentage = isReadOnly ? 0.4 : 0.25
const maxPercentage = isReadOnly ? 0.5 : 0.3
const maxItems = Math.ceil(totalItems * maxPercentage)
// Return the smaller of the two to avoid excessive memory usage
return Math.min(browserCacheSize, maxItems)
return Math.min(optimalSize, maxItems)
}
return browserCacheSize
return optimalSize
} catch (error) {
console.warn('Failed to detect optimal cache size:', error)
return defaultSize
}
// For worker environments or when memory detection fails
if (this.environment === Environment.WORKER) {
// Workers typically have limited memory, be conservative
return isReadOnly ? 2000 : 1000
}
return defaultSize
} catch (error) {
console.warn('Error detecting optimal cache size:', error)
return 1000 // Conservative default
@ -448,82 +338,31 @@ export class CacheManager<T extends HNSWNode | Edge | HNSWVerb> {
}
/**
* Detects available memory across different environments
*
* This method uses different techniques to detect memory in:
* - Node.js: Uses the OS module with dynamic import
* - Browser: Uses performance.memory or navigator.deviceMemory
* - Worker: Uses performance.memory if available
*
* Detect available memory using Node-like runtime primitives.
*
* Brainy 8.0 only runs on Node.js, Bun, and Deno, so this method reads real
* values from `node:os` (`process.memoryUsage()` is not used here because we
* want system-wide free memory, not just the V8 heap). If the import fails,
* we fall back to conservative defaults that match the synchronous path.
*
* @returns An object with totalMemory and freeMemory in bytes, or null if detection fails
*/
private async detectAvailableMemory(): Promise<{ totalMemory: number, freeMemory: number } | null> {
try {
// Node.js environment
if (this.environment === Environment.NODE) {
try {
// Use dynamic import for OS module
const os = await import('node:os')
// Get actual system memory information
const totalMemory = os.totalmem()
const freeMemory = os.freemem()
return { totalMemory, freeMemory }
} catch (error) {
console.warn('Failed to detect memory in Node.js environment:', error)
}
try {
// Use dynamic import for OS module
const os = await import('node:os')
// Get actual system memory information
const totalMemory = os.totalmem()
const freeMemory = os.freemem()
return { totalMemory, freeMemory }
} catch (error) {
console.warn('Failed to detect memory via node:os:', error)
}
// Browser environment
if (this.environment === Environment.BROWSER) {
// Try using performance.memory (Chrome only)
if (performance && (performance as any).memory) {
const memoryInfo = (performance as any).memory
// jsHeapSizeLimit is the maximum size of the heap
// totalJSHeapSize is the currently allocated heap size
// usedJSHeapSize is the amount of heap currently being used
const totalMemory = memoryInfo.jsHeapSizeLimit || 0
const usedMemory = memoryInfo.usedJSHeapSize || 0
const freeMemory = Math.max(totalMemory - usedMemory, 0)
return { totalMemory, freeMemory }
}
// Try using navigator.deviceMemory as fallback
if (navigator.deviceMemory) {
// deviceMemory is in GB, convert to bytes
const totalMemory = navigator.deviceMemory * 1024 * 1024 * 1024
// Assume 50% is free
const freeMemory = totalMemory * 0.5
return { totalMemory, freeMemory }
}
}
// Worker environment
if (this.environment === Environment.WORKER) {
// Try using performance.memory if available (Chrome workers)
if (performance && (performance as any).memory) {
const memoryInfo = (performance as any).memory
const totalMemory = memoryInfo.jsHeapSizeLimit || 0
const usedMemory = memoryInfo.usedJSHeapSize || 0
const freeMemory = Math.max(totalMemory - usedMemory, 0)
return { totalMemory, freeMemory }
}
// For workers, use a conservative estimate
// Assume 2GB total memory with 1GB free
return {
totalMemory: 2 * 1024 * 1024 * 1024,
freeMemory: 1 * 1024 * 1024 * 1024
}
}
// If all detection methods fail, use conservative defaults
// If detection failed, use conservative defaults
return {
totalMemory: 8 * 1024 * 1024 * 1024, // Assume 8GB total
freeMemory: 4 * 1024 * 1024 * 1024 // Assume 4GB free