soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: exclude __words__ keyword index from corruption detection and getStats()

Browse source 
The __words__ keyword index stores 50-5000 entries per entity (one per
word), which inflated avg entries/entity well above the corruption
threshold of 100. This caused:

1. validateConsistency() to falsely detect corruption on every startup,
   triggering unnecessary clearAllIndexData() + rebuild() cycles
2. getStats() to log false "Metadata index may be corrupted" warnings
   and report inflated totalEntries/totalIds stats

Both methods now skip __words__ when counting, so stats and health
checks reflect metadata fields only (noun, type, createdAt, etc.).
Keyword search is unaffected since the __words__ field index itself
is not modified.
This commit is contained in:
David Snelling 2026-01-27 15:38:21 -08:00
parent 32dbdcec61
commit 364360d447
128 changed files with 5637 additions and 5682 deletions
Download patch file Download diff file Expand all files Collapse all files

108
src/storage/adapters/azureBlobStorage.ts
View file

@ -9,7 +9,7 @@
* 4. SAS Token
* 5. Azure AD (OAuth2) via DefaultAzureCredential
*
* v4.0.0: Fully compatible with metadata/vector separation architecture
* Fully compatible with metadata/vector separation architecture
*/
import {
@ -65,7 +65,7 @@ const MAX_AZURE_PAGE_SIZE = 5000
* 3. Storage Account Key - if accountName + accountKey provided
* 4. SAS Token - if accountName + sasToken provided
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed pagination overrides
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -105,7 +105,7 @@ export class AzureBlobStorage extends BaseStorage {
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
// v6.2.7: Write buffering always enabled for consistent performance
// Write buffering always enabled for consistent performance
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
// Multi-level cache manager for efficient data access
@ -115,7 +115,7 @@ export class AzureBlobStorage extends BaseStorage {
// Module logger
private logger = createModuleLogger('AzureBlobStorage')
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
// HNSW mutex locks to prevent read-modify-write races
private hnswLocks = new Map<string, Promise<void>>()
/**
@ -162,7 +162,7 @@ export class AzureBlobStorage extends BaseStorage {
}
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
* strict locally. Zero-config optimization.
@ -171,7 +171,6 @@ export class AzureBlobStorage extends BaseStorage {
* - `'strict'`: Traditional blocking init. Validates container and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: InitMode
@ -185,7 +184,7 @@ export class AzureBlobStorage extends BaseStorage {
this.sasToken = options.sasToken
this.readOnly = options.readOnly || false
// v7.3.0: Handle initMode
// Handle initMode
if (options.initMode) {
this.initMode = options.initMode
}
@ -201,7 +200,7 @@ export class AzureBlobStorage extends BaseStorage {
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// v6.2.7: Write buffering always enabled - no env var check needed
// Write buffering always enabled - no env var check needed
}
/**
@ -216,7 +215,7 @@ export class AzureBlobStorage extends BaseStorage {
* Recent Azure improvements make parallel downloads very efficient
*
* @returns Azure Blob-optimized batch configuration
* @since v5.12.0 - Updated for native batch API
* Updated for native batch API
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -241,7 +240,6 @@ export class AzureBlobStorage extends BaseStorage {
*
* @param paths - Array of Azure blob paths to read
* @returns Map of path -> parsed JSON data (only successful reads)
* @since v5.12.0
*/
public async readBatch(paths: string[]): Promise<Map<string, any>> {
await this.ensureInitialized()
@ -297,7 +295,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Initialize the storage adapter
*
* v7.3.0: Supports progressive initialization for fast cold starts
* Supports progressive initialization for fast cold starts
*
* | Mode | Init Time | When |
* |------|-----------|------|
@ -404,7 +402,7 @@ export class AzureBlobStorage extends BaseStorage {
this.verbCacheManager.clear()
prodLog.info('✅ Cache cleared - starting fresh')
// v7.3.0: Progressive vs Strict initialization
// Progressive vs Strict initialization
if (effectiveMode === 'progressive') {
// PROGRESSIVE MODE: Fast init, background validation
// Mark as initialized immediately - ready to accept operations
@ -413,7 +411,7 @@ export class AzureBlobStorage extends BaseStorage {
prodLog.info(`✅ Azure progressive init complete: ${this.containerName} (validation deferred)`)
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Schedule background tasks (non-blocking)
@ -434,7 +432,7 @@ export class AzureBlobStorage extends BaseStorage {
await this.initializeCounts()
this.countsLoaded = true
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Mark background tasks as complete (nothing to do in background)
@ -447,7 +445,7 @@ export class AzureBlobStorage extends BaseStorage {
}
// =============================================
// Progressive Initialization (v7.3.0+)
// Progressive Initialization
// =============================================
/**
@ -461,7 +459,6 @@ export class AzureBlobStorage extends BaseStorage {
*
* @protected
* @override
* @since v7.3.0
*/
protected async runBackgroundInit(): Promise<void> {
const startTime = Date.now()
@ -485,7 +482,6 @@ export class AzureBlobStorage extends BaseStorage {
* bucketValidated/bucketValidationError for lazy use.
*
* @private
* @since v7.3.0
*/
private async validateContainerInBackground(): Promise<void> {
try {
@ -517,7 +513,6 @@ export class AzureBlobStorage extends BaseStorage {
* Load counts from storage in background.
*
* @private
* @since v7.3.0
*/
private async loadCountsInBackground(): Promise<void> {
try {
@ -541,7 +536,6 @@ export class AzureBlobStorage extends BaseStorage {
* @throws Error if container validation fails
* @protected
* @override
* @since v7.3.0
*/
protected async ensureValidatedForWrite(): Promise<void> {
// If already validated, nothing to do
@ -665,7 +659,7 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
/**
* Flush noun buffer to Azure
@ -697,20 +691,20 @@ export class AzureBlobStorage extends BaseStorage {
await Promise.all(writes)
}
// v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Save a node to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.nounWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
@ -791,7 +785,7 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.4.0: Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Get a node from storage
@ -889,18 +883,18 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.4.0: Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Write an object to a specific path in Azure
* Primitive operation required by base class
*
* v7.3.0: Performs lazy container validation on first write in progressive mode.
* Performs lazy container validation on first write in progressive mode.
* @protected
*/
protected async writeObjectToPath(path: string, data: any): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy container validation for progressive init
// Lazy container validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -954,12 +948,12 @@ export class AzureBlobStorage extends BaseStorage {
* Delete an object from a specific path in Azure
* Primitive operation required by base class
*
* v7.3.0: Performs lazy container validation on first delete in progressive mode.
* Performs lazy container validation on first delete in progressive mode.
* @protected
*/
protected async deleteObjectFromPath(path: string): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy container validation for progressive init
// Lazy container validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -1209,20 +1203,20 @@ export class AzureBlobStorage extends BaseStorage {
})
}
// v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
/**
* Save an edge to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.verbWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
this.verbCacheManager.set(edge.id, edge)
await this.verbWriteBuffer.add(edge.id, edge)
@ -1255,7 +1249,7 @@ export class AzureBlobStorage extends BaseStorage {
])
),
// CORE RELATIONAL DATA (v4.0.0)
// CORE RELATIONAL DATA
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
@ -1280,7 +1274,7 @@ export class AzureBlobStorage extends BaseStorage {
// Update cache
this.verbCacheManager.set(edge.id, edge)
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveVerbMetadata_internal
// This fixes the race condition where metadata didn't exist yet
this.logger.trace(`Edge ${edge.id} saved successfully`)
@ -1298,7 +1292,7 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.4.0: Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
/**
* Get an edge from storage
@ -1335,7 +1329,7 @@ export class AzureBlobStorage extends BaseStorage {
connections.set(Number(level), new Set(verbIds as string[]))
}
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
// Return HNSWVerb with core relational fields (NO metadata field)
const edge: Edge = {
id: data.id,
vector: data.vector,
@ -1346,7 +1340,7 @@ export class AzureBlobStorage extends BaseStorage {
sourceId: data.sourceId,
targetId: data.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
}
@ -1375,13 +1369,13 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.4.0: Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed getNounsWithPagination - now inherit from BaseStorage's type-first implementation
// Removed getNounsWithPagination - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed getNounsByNounType_internal - now inherit from BaseStorage's type-first implementation
// Removed getNounsByNounType_internal - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed 3 verb query *_internal methods (getVerbsBySource, getVerbsByTarget, getVerbsByType) - now inherit from BaseStorage's type-first implementation
// Removed 3 verb query *_internal methods (getVerbsBySource, getVerbsByTarget, getVerbsByType) - now inherit from BaseStorage's type-first implementation
/**
* Clear all data from storage
@ -1393,7 +1387,7 @@ export class AzureBlobStorage extends BaseStorage {
this.logger.info('🧹 Clearing all data from Azure container...')
// Delete all blobs in container
// v5.6.1: listBlobsFlat() returns ALL blobs including _cow/ prefix
// listBlobsFlat() returns ALL blobs including _cow/ prefix
// This correctly deletes COW version control data (commits, trees, blobs, refs)
for await (const blob of this.containerClient!.listBlobsFlat()) {
if (blob.name) {
@ -1402,7 +1396,7 @@ export class AzureBlobStorage extends BaseStorage {
}
}
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -1461,12 +1455,12 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker blob exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -1643,7 +1637,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Get a noun's vector for HNSW rebuild
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
@ -1653,7 +1647,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Uses mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -1662,7 +1656,7 @@ export class AzureBlobStorage extends BaseStorage {
}): Promise<void> {
const lockKey = `hnsw/${nounId}`
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
// Problem: Without mutex, concurrent operations can:
// 1. Thread A reads noun (connections: [1,2,3])
// 2. Thread B reads noun (connections: [1,2,3])
@ -1682,7 +1676,7 @@ export class AzureBlobStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
// Read existing noun data (if exists)
const existingNoun = await this.getNoun(nounId)
@ -1704,7 +1698,7 @@ export class AzureBlobStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
await this.saveNoun(updatedNoun)
} finally {
// Release lock (ALWAYS runs, even if error thrown)
@ -1715,7 +1709,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -1744,7 +1738,7 @@ export class AzureBlobStorage extends BaseStorage {
/**
* Save HNSW system data (entry point, max level)
*
* CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions
* CRITICAL FIX: Optimistic locking with ETags to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
@ -1840,7 +1834,7 @@ export class AzureBlobStorage extends BaseStorage {
}
/**
* Set the access tier for a specific blob (v4.0.0 cost optimization)
* Set the access tier for a specific blob (cost optimization)
* Azure Blob Storage tiers:
* - Hot: $0.0184/GB/month - Frequently accessed data
* - Cool: $0.01/GB/month - Infrequently accessed data (45% cheaper)
@ -1906,7 +1900,7 @@ export class AzureBlobStorage extends BaseStorage {
}
/**
* Set access tier for multiple blobs in batch (v4.0.0 cost optimization)
* Set access tier for multiple blobs in batch (cost optimization)
* Efficiently move large numbers of blobs between tiers for cost optimization
*
* @param blobs - Array of blob names and their target tiers
@ -2128,7 +2122,7 @@ export class AzureBlobStorage extends BaseStorage {
}
/**
* Set lifecycle management policy for automatic tier transitions and deletions (v4.0.0)
* Set lifecycle management policy for automatic tier transitions and deletions
* Automates cost optimization by moving old data to cheaper tiers or deleting it
*
* Azure Lifecycle Management rules run once per day and apply to the entire container.

43
src/storage/adapters/baseStorageAdapter.ts
View file

@ -19,7 +19,7 @@ import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/field
import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
// =============================================================================
// Progressive Initialization Types (v7.3.0+)
// Progressive Initialization Types
// =============================================================================
/**
@ -29,7 +29,6 @@ import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
* fast cold starts in serverless environments while maintaining strict
* validation for local development.
*
* @since v7.3.0
*
* | Mode | Description | Use Case |
* |------|-------------|----------|
@ -96,7 +95,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
abstract getVerbMetadata(id: string): Promise<any | null>
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
// These methods enable HNSW index rebuilding after container restarts
abstract getNounVector(id: string): Promise<number[] | null>
@ -139,7 +138,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* relateMany(), and import operations to automatically adapt to storage capabilities.
*
* @returns Batch configuration optimized for this storage type
* @since v4.11.0
*/
public getBatchConfig(): StorageBatchConfig {
// Conservative defaults that work safely across all storage types
@ -295,7 +293,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}> = new Map()
// =============================================
// Progressive Initialization State (v7.3.0+)
// Progressive Initialization State
// =============================================
// These properties enable fast cold starts in cloud environments
// by deferring validation and count loading to background tasks.
@ -308,7 +306,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* - `'strict'`: Always validate during init (traditional behavior)
*
* @protected
* @since v7.3.0
*/
protected initMode: InitMode = 'auto'
@ -319,7 +316,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* after the first successful write operation or background validation.
*
* @protected
* @since v7.3.0
*/
protected bucketValidated = false
@ -329,7 +325,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* Stored here so subsequent operations can fail fast with the same error.
*
* @protected
* @since v7.3.0
*/
protected bucketValidationError: Error | null = null
@ -340,7 +335,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* Operations work immediately; counts become accurate after background load.
*
* @protected
* @since v7.3.0
*/
protected countsLoaded = false
@ -350,7 +344,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* Useful for tests and diagnostics to ensure full initialization.
*
* @protected
* @since v7.3.0
*/
protected backgroundTasksComplete = false
@ -361,7 +354,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* before proceeding (e.g., in tests).
*
* @protected
* @since v7.3.0
*/
protected backgroundInitPromise: Promise<void> | null = null
@ -1057,7 +1049,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
protected readonly COUNT_CACHE_TTL = 60000 // 1 minute cache TTL
// =============================================
// Smart Count Batching (v3.32.3+)
// Smart Count Batching
// =============================================
// Count batching state - mirrors statistics batching pattern
@ -1112,7 +1104,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-entity-${type}`, async () => {
this.incrementEntityCount(type)
// Smart batching (v3.32.3+): Adapts to storage type
// Smart batching: Adapts to storage type
// - Cloud storage (GCS, S3): Batches 10 ops OR 5 seconds
// - Local storage (File, Memory): Persists immediately
await this.scheduleCountPersist()
@ -1147,13 +1139,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-entity-${type}`, async () => {
this.decrementEntityCount(type)
// Smart batching (v3.32.3+): Adapts to storage type
// Smart batching: Adapts to storage type
await this.scheduleCountPersist()
})
}
/**
* Increment verb count - O(1) operation (v4.1.2: now synchronous)
* Increment verb count - O(1) operation (now synchronous)
* Protected by storage-specific mechanisms (mutex, distributed consensus, etc.)
* @param type The verb type
*/
@ -1168,7 +1160,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
/**
* Thread-safe increment for verb counts (v4.1.2)
* Thread-safe increment for verb counts
* Uses mutex for single-node, distributed consensus for multi-node
* @param type The verb type
*/
@ -1176,13 +1168,13 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-verb-${type}`, async () => {
this.incrementVerbCount(type)
// Smart batching (v3.32.3+): Adapts to storage type
// Smart batching: Adapts to storage type
await this.scheduleCountPersist()
})
}
/**
* Decrement verb count - O(1) operation (v4.1.2: now synchronous)
* Decrement verb count - O(1) operation (now synchronous)
* @param type The verb type
*/
protected decrementVerbCount(type: string): void {
@ -1203,20 +1195,20 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
/**
* Thread-safe decrement for verb counts (v4.1.2)
* Thread-safe decrement for verb counts
* @param type The verb type
*/
protected async decrementVerbCountSafe(type: string): Promise<void> {
const mutex = getGlobalMutex()
await mutex.runExclusive(`count-verb-${type}`, async () => {
this.decrementVerbCount(type)
// Smart batching (v3.32.3+): Adapts to storage type
// Smart batching: Adapts to storage type
await this.scheduleCountPersist()
})
}
// =============================================
// Smart Batching Methods (v3.32.3+)
// Smart Batching Methods
// =============================================
/**
@ -1235,7 +1227,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
}
// =============================================
// Progressive Initialization Methods (v7.3.0+)
// Progressive Initialization Methods
// =============================================
/**
@ -1254,7 +1246,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
*
* @returns `true` if running in a detected cloud environment
* @protected
* @since v7.3.0
*/
protected detectCloudEnvironment(): boolean {
return !!(
@ -1274,7 +1265,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
*
* @returns The resolved init mode (`'progressive'` or `'strict'`)
* @protected
* @since v7.3.0
*/
protected resolveInitMode(): 'progressive' | 'strict' {
if (this.initMode === 'auto') {
@ -1295,7 +1285,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* Always call `super.scheduleBackgroundInit()` first.
*
* @protected
* @since v7.3.0
*/
protected scheduleBackgroundInit(): void {
// Create a promise that tracks all background work
@ -1320,7 +1309,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* The default implementation does nothing (for local storage adapters).
*
* @protected
* @since v7.3.0
*/
protected async runBackgroundInit(): Promise<void> {
// Default implementation: nothing to do for local storage
@ -1348,7 +1336,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
* ```
*
* @public
* @since v7.3.0
*/
public async awaitBackgroundInit(): Promise<void> {
if (this.backgroundInitPromise) {
@ -1361,7 +1348,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
*
* @returns `true` if all background tasks are complete
* @public
* @since v7.3.0
*/
public isBackgroundInitComplete(): boolean {
return this.backgroundTasksComplete
@ -1379,7 +1365,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
*
* @throws Error if bucket validation fails
* @protected
* @since v7.3.0
*/
protected async ensureValidatedForWrite(): Promise<void> {
// If already validated, nothing to do

109
src/storage/adapters/fileSystemStorage.ts
View file

@ -59,7 +59,7 @@ try {
* File system storage adapter for Node.js environments
* Uses the file system to store data in the specified directory structure
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -91,12 +91,12 @@ export class FileSystemStorage extends BaseStorage {
private lockTimers: Map<string, NodeJS.Timeout> = new Map() // Track timers for cleanup
private allTimers: Set<NodeJS.Timeout> = new Set() // Track all timers for cleanup
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
// CRITICAL FIX: Mutex locks for HNSW concurrency control
// Prevents read-modify-write races during concurrent neighbor updates at scale (1000+ ops)
// Matches MemoryStorage and OPFSStorage behavior (tested in production)
private hnswLocks = new Map<string, Promise<void>>()
// Compression configuration (v4.0.0)
// Compression configuration
private compressionEnabled: boolean = true // Enable gzip compression by default for 60-80% disk savings
private compressionLevel: number = 6 // zlib compression level (1-9, default: 6 = balanced)
@ -136,7 +136,6 @@ export class FileSystemStorage extends BaseStorage {
* - Parallel processing supported
*
* @returns FileSystem-optimized batch configuration
* @since v4.11.0
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -179,7 +178,7 @@ export class FileSystemStorage extends BaseStorage {
try {
// Initialize directory paths now that path module is loaded
// Clean directory structure (v4.7.2+)
// Clean directory structure
this.nounsDir = path.join(this.rootDir, 'entities/nouns/hnsw')
this.verbsDir = path.join(this.rootDir, 'entities/verbs/hnsw')
this.metadataDir = path.join(this.rootDir, 'entities/nouns/metadata') // Legacy reference
@ -245,7 +244,7 @@ export class FileSystemStorage extends BaseStorage {
// Always use fixed depth after migration/detection
this.cachedShardingDepth = this.SHARDING_DEPTH
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
} catch (error) {
console.error('Error initializing FileSystemStorage:', error)
@ -281,7 +280,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Save a node to storage
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
@ -303,7 +302,7 @@ export class FileSystemStorage extends BaseStorage {
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
try {
// ATOMIC WRITE SEQUENCE (v4.10.3):
// ATOMIC WRITE SEQUENCE:
// 1. Write to temp file
await this.ensureDirectoryExists(path.dirname(tempPath))
await fs.promises.writeFile(tempPath, JSON.stringify(serializableNode, null, 2))
@ -320,7 +319,7 @@ export class FileSystemStorage extends BaseStorage {
throw error
}
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveNounMetadata_internal
// This fixes the race condition where metadata didn't exist yet
}
@ -361,7 +360,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Get all nodes from storage
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
* Previously only scanned flat directory, causing rebuild to find 0 entities
*/
protected async getAllNodes(): Promise<HNSWNode[]> {
@ -406,7 +405,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Get nodes by noun type
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
* @param nounType The noun type to filter by
* @returns Promise that resolves to an array of nodes of the specified noun type
*/
@ -462,7 +461,7 @@ export class FileSystemStorage extends BaseStorage {
const filePath = this.getNodePath(id)
// Load metadata to get type for count update (v4.0.0: separate storage)
// Load metadata to get type for count update (separate storage)
try {
const metadata = await this.getNounMetadata(id)
if (metadata) {
@ -490,13 +489,13 @@ export class FileSystemStorage extends BaseStorage {
/**
* Save an edge to storage
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// Convert connections Map to a serializable format
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
@ -505,7 +504,7 @@ export class FileSystemStorage extends BaseStorage {
Array.from(set as Set<string>)
),
// CORE RELATIONAL DATA (v3.50.1+)
// CORE RELATIONAL DATA
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
@ -518,7 +517,7 @@ export class FileSystemStorage extends BaseStorage {
const tempPath = `${filePath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
try {
// ATOMIC WRITE SEQUENCE (v4.10.3):
// ATOMIC WRITE SEQUENCE:
// 1. Write to temp file
await this.ensureDirectoryExists(path.dirname(tempPath))
await fs.promises.writeFile(tempPath, JSON.stringify(serializableEdge, null, 2))
@ -535,7 +534,7 @@ export class FileSystemStorage extends BaseStorage {
throw error
}
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveVerbMetadata_internal
// This fixes the race condition where metadata didn't exist yet
}
@ -556,7 +555,7 @@ export class FileSystemStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
// Return HNSWVerb with core relational fields (NO metadata field)
return {
id: parsedEdge.id,
vector: parsedEdge.vector,
@ -567,7 +566,7 @@ export class FileSystemStorage extends BaseStorage {
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
}
} catch (error: any) {
@ -580,7 +579,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Get all edges from storage
* CRITICAL FIX (v3.43.2): Now scans sharded subdirectories (depth=1)
* CRITICAL FIX: Now scans sharded subdirectories (depth=1)
* Previously only scanned flat directory, causing rebuild to find 0 relationships
*/
protected async getAllEdges(): Promise<Edge[]> {
@ -608,7 +607,7 @@ export class FileSystemStorage extends BaseStorage {
connections.set(Number(level), new Set(nodeIds as string[]))
}
// v4.0.0: Include core relational fields (NO metadata field)
// Include core relational fields (NO metadata field)
allEdges.push({
id: parsedEdge.id,
vector: parsedEdge.vector,
@ -619,7 +618,7 @@ export class FileSystemStorage extends BaseStorage {
sourceId: parsedEdge.sourceId,
targetId: parsedEdge.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
})
}
@ -696,8 +695,8 @@ export class FileSystemStorage extends BaseStorage {
/**
* Primitive operation: Write object to path
* All metadata operations use this internally via base class routing
* v4.0.0: Supports gzip compression for 60-80% disk savings
* CRITICAL FIX (v4.10.3): Added atomic write pattern to prevent file corruption during concurrent imports
* Supports gzip compression for 60-80% disk savings
* CRITICAL FIX: Added atomic write pattern to prevent file corruption during concurrent imports
*/
protected async writeObjectToPath(pathStr: string, data: any): Promise<void> {
await this.ensureInitialized()
@ -711,7 +710,7 @@ export class FileSystemStorage extends BaseStorage {
const tempPath = `${compressedPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
try {
// ATOMIC WRITE SEQUENCE (v4.10.3):
// ATOMIC WRITE SEQUENCE:
// 1. Compress and write to temp file
const jsonString = JSON.stringify(data, null, 2)
const compressed = await new Promise<Buffer>((resolve, reject) => {
@ -748,7 +747,7 @@ export class FileSystemStorage extends BaseStorage {
const tempPath = `${fullPath}.tmp.${Date.now()}.${Math.random().toString(36).substring(2)}`
try {
// ATOMIC WRITE SEQUENCE (v4.10.3):
// ATOMIC WRITE SEQUENCE:
// 1. Write to temp file
await fs.promises.writeFile(tempPath, JSON.stringify(data, null, 2))
@ -770,7 +769,7 @@ export class FileSystemStorage extends BaseStorage {
* Primitive operation: Read object from path
* All metadata operations use this internally via base class routing
* Enhanced error handling for corrupted metadata files (Bug #3 mitigation)
* v4.0.0: Supports reading both compressed (.gz) and uncompressed files for backward compatibility
* Supports reading both compressed (.gz) and uncompressed files for backward compatibility
*/
protected async readObjectFromPath(pathStr: string): Promise<any | null> {
await this.ensureInitialized()
@ -822,7 +821,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Primitive operation: Delete object from path
* All metadata operations use this internally via base class routing
* v4.0.0: Deletes both compressed and uncompressed versions (for cleanup)
* Deletes both compressed and uncompressed versions (for cleanup)
*/
protected async deleteObjectFromPath(pathStr: string): Promise<void> {
await this.ensureInitialized()
@ -863,7 +862,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Primitive operation: List objects under path prefix
* All metadata operations use this internally via base class routing
* v4.0.0: Handles both .json and .json.gz files, normalizes paths
* Handles both .json and .json.gz files, normalizes paths
*/
protected async listObjectsUnderPath(prefix: string): Promise<string[]> {
await this.ensureInitialized()
@ -877,7 +876,7 @@ export class FileSystemStorage extends BaseStorage {
for (const entry of entries) {
if (entry.isFile()) {
// v5.3.5: Handle multiple compression formats for broad compatibility
// Handle multiple compression formats for broad compatibility
// - .json.gz: Standard entity/metadata files (JSON compressed)
// - .gz: COW files (refs, blobs, commits - raw compressed)
// - .json: Uncompressed JSON files
@ -890,7 +889,7 @@ export class FileSystemStorage extends BaseStorage {
seen.add(normalizedPath)
}
} else if (entry.name.endsWith('.gz')) {
// v5.3.5 fix: COW files stored as .gz (not .json.gz)
// COW files stored as .gz (not .json.gz)
// Strip .gz extension and return path
const normalizedName = entry.name.slice(0, -3) // Remove .gz
const normalizedPath = path.join(prefix, normalizedName)
@ -966,7 +965,7 @@ export class FileSystemStorage extends BaseStorage {
* Get nouns with pagination support
* @param options Pagination options
*/
// v5.4.0: Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation
// Removed getNounsWithPagination override - now uses BaseStorage's type-first implementation
/**
* Clear all data from storage
@ -1002,7 +1001,7 @@ export class FileSystemStorage extends BaseStorage {
}
}
// v5.10.4: Clear the entire branches/ directory (branch-based storage)
// Clear the entire branches/ directory (branch-based storage)
// Bug fix: Data is stored in branches/main/entities/, not just entities/
// The branch-based structure was introduced for COW support
const branchesDir = path.join(this.rootDir, 'branches')
@ -1016,7 +1015,7 @@ export class FileSystemStorage extends BaseStorage {
await removeDirectoryContents(this.indexDir)
}
// v5.6.1: Remove COW (copy-on-write) version control data
// Remove COW (copy-on-write) version control data
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
// Must be deleted to fully clear all data including version history
const cowDir = path.join(this.rootDir, '_cow')
@ -1024,7 +1023,7 @@ export class FileSystemStorage extends BaseStorage {
// Delete the entire _cow/ directory (not just contents)
await fs.promises.rm(cowDir, { recursive: true, force: true })
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -1035,12 +1034,12 @@ export class FileSystemStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
// Reset entity counters (inherited from BaseStorageAdapter)
// These in-memory counters must be reset to 0 after clearing all data
;(this as any).totalNounCount = 0
;(this as any).totalVerbCount = 0
// v7.3.1: Clear write-through cache (inherited from BaseStorage)
// Clear write-through cache (inherited from BaseStorage)
// Without this, readWithInheritance() would return stale cached data
// after clear(), causing "ghost" entities to appear
this.clearWriteCache()
@ -1074,12 +1073,12 @@ export class FileSystemStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker file exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -1152,7 +1151,7 @@ export class FileSystemStorage extends BaseStorage {
totalSize = nounsDirSize + verbsDirSize + metadataDirSize + indexDirSize
// CRITICAL FIX (v3.43.2): Use persisted counts instead of directory reads
// CRITICAL FIX: Use persisted counts instead of directory reads
// This is O(1) instead of O(n), and handles sharded structure correctly
const nounsCount = this.totalNounCount
const verbsCount = this.totalVerbCount
@ -1210,8 +1209,8 @@ export class FileSystemStorage extends BaseStorage {
}
}
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
// Removed 2 pagination methods (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's implementation
/**
* Acquire a file-based lock for coordinating operations across multiple processes
@ -1503,14 +1502,14 @@ export class FileSystemStorage extends BaseStorage {
this.totalVerbCount = validVerbFiles.length
// Sample some files to get type distribution (don't read all)
// v4.0.0: Load metadata separately for type information
// Load metadata separately for type information
const sampleSize = Math.min(100, validNounFiles.length)
for (let i = 0; i < sampleSize; i++) {
try {
const file = validNounFiles[i]
const id = file.replace('.json', '')
// v4.0.0: Load metadata from separate storage for type info
// Load metadata from separate storage for type info
const metadata = await this.getNounMetadata(id)
if (metadata) {
const type = metadata.noun || 'default'
@ -2148,7 +2147,7 @@ export class FileSystemStorage extends BaseStorage {
const edge = JSON.parse(data)
const metadata = await this.getVerbMetadata(id)
// v4.8.1: Don't skip verbs without metadata - metadata is optional
// Don't skip verbs without metadata - metadata is optional
// FIX: This was the root cause of the VFS bug (11 versions)
// Verbs can exist without metadata files (e.g., from imports/migrations)
@ -2162,7 +2161,7 @@ export class FileSystemStorage extends BaseStorage {
connections = connectionsMap
}
// v4.8.0: Extract standard fields from metadata to top-level
// Extract standard fields from metadata to top-level
const metadataObj = (metadata || {}) as VerbMetadata
const { createdAt, updatedAt, confidence, weight, service, data: dataField, createdBy, ...customMetadata } = metadataObj
@ -2309,12 +2308,12 @@ export class FileSystemStorage extends BaseStorage {
}
// =============================================
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
// =============================================
/**
* Get vector for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
@ -2324,7 +2323,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -2333,7 +2332,7 @@ export class FileSystemStorage extends BaseStorage {
}): Promise<void> {
const lockKey = `hnsw/${nounId}`
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
// Problem: Without mutex, concurrent operations can:
// 1. Thread A reads noun (connections: [1,2,3])
// 2. Thread B reads noun (connections: [1,2,3])
@ -2353,7 +2352,7 @@ export class FileSystemStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
// Read existing noun data (if exists)
const existingNoun = await this.getNoun(nounId)
@ -2375,7 +2374,7 @@ export class FileSystemStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write)
// Use BaseStorage's saveNoun (type-first paths, atomic write)
await this.saveNoun(updatedNoun)
} finally {
// Release lock (ALWAYS runs, even if error thrown)
@ -2386,7 +2385,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -2415,7 +2414,7 @@ export class FileSystemStorage extends BaseStorage {
/**
* Save HNSW system data (entry point, max level)
*
* CRITICAL FIX (v4.10.1): Mutex lock + atomic write to prevent race conditions
* CRITICAL FIX: Mutex lock + atomic write to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
@ -2425,7 +2424,7 @@ export class FileSystemStorage extends BaseStorage {
const lockKey = 'hnsw/system'
// CRITICAL FIX (v4.10.1): Mutex lock to serialize system updates
// CRITICAL FIX: Mutex lock to serialize system updates
// System data (entry point, max level) updated frequently during HNSW construction
// Without mutex, concurrent updates can lose data (same as entity-level problem)

135
src/storage/adapters/gcsStorage.ts
View file

@ -65,7 +65,7 @@ const MAX_GCS_PAGE_SIZE = 5000
* 3. Service Account Credentials Object (if credentials provided)
* 4. HMAC Keys (if accessKeyId/secretAccessKey provided)
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -110,7 +110,7 @@ export class GcsStorage extends BaseStorage {
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
// v6.2.7: Write buffering always enabled for consistent performance
// Write buffering always enabled for consistent performance
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
// Multi-level cache manager for efficient data access
@ -120,7 +120,7 @@ export class GcsStorage extends BaseStorage {
// Module logger
private logger = createModuleLogger('GcsStorage')
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
// HNSW mutex locks to prevent read-modify-write races
private hnswLocks = new Map<string, Promise<void>>()
// Configuration options
@ -166,7 +166,7 @@ export class GcsStorage extends BaseStorage {
secretAccessKey?: string
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Cloud Run, Lambda),
* strict locally. Zero-config optimization.
@ -175,19 +175,18 @@ export class GcsStorage extends BaseStorage {
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: InitMode
/**
* @deprecated Use `initMode: 'progressive'` instead.
* Will be removed in v8.0.0.
* Will be removed in a future version.
*/
skipInitialScan?: boolean
/**
* @deprecated Use `initMode: 'progressive'` instead.
* Will be removed in v8.0.0.
* Will be removed in a future version.
*/
skipCountsFile?: boolean
@ -206,7 +205,7 @@ export class GcsStorage extends BaseStorage {
this.accessKeyId = options.accessKeyId
this.secretAccessKey = options.secretAccessKey
// v7.3.0: Handle initMode and deprecated skip* flags
// Handle initMode and deprecated skip* flags
if (options.initMode) {
this.initMode = options.initMode
}
@ -215,14 +214,14 @@ export class GcsStorage extends BaseStorage {
if (options.skipInitialScan) {
console.warn(
'[GcsStorage] DEPRECATION WARNING: skipInitialScan is deprecated. ' +
'Use initMode: "progressive" instead. Will be removed in v8.0.0.'
'Use initMode: "progressive" instead. Will be removed in a future version.'
)
this.skipInitialScan = true
}
if (options.skipCountsFile) {
console.warn(
'[GcsStorage] DEPRECATION WARNING: skipCountsFile is deprecated. ' +
'Use initMode: "progressive" instead. Will be removed in v8.0.0.'
'Use initMode: "progressive" instead. Will be removed in a future version.'
)
this.skipCountsFile = true
}
@ -240,13 +239,13 @@ export class GcsStorage extends BaseStorage {
this.nounCacheManager = new CacheManager<HNSWNode>(options.cacheConfig)
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// v6.2.7: Write buffering always enabled - no env var check needed
// Write buffering always enabled - no env var check needed
}
/**
* Initialize the storage adapter
*
* v7.3.0: Supports progressive initialization for fast cold starts
* Supports progressive initialization for fast cold starts
*
* | Mode | Init Time | When |
* |------|-----------|------|
@ -337,14 +336,14 @@ export class GcsStorage extends BaseStorage {
}
)
// CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs
// CRITICAL FIX: Clear any stale cache entries from previous runs
// This prevents cache poisoning from causing silent failures on container restart
prodLog.info('🧹 Clearing cache from previous run to prevent cache poisoning')
this.nounCacheManager.clear()
this.verbCacheManager.clear()
prodLog.info('✅ Cache cleared - starting fresh')
// v7.3.0: Progressive vs Strict initialization
// Progressive vs Strict initialization
if (effectiveMode === 'progressive') {
// PROGRESSIVE MODE: Fast init, background validation
// Mark as initialized immediately - ready to accept operations
@ -353,7 +352,7 @@ export class GcsStorage extends BaseStorage {
prodLog.info(`✅ GCS progressive init complete: ${this.bucketName} (validation deferred)`)
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Schedule background tasks (non-blocking)
@ -373,7 +372,7 @@ export class GcsStorage extends BaseStorage {
await this.initializeCounts()
this.countsLoaded = true
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Mark background tasks as complete (nothing to do in background)
@ -386,7 +385,7 @@ export class GcsStorage extends BaseStorage {
}
// =============================================
// Progressive Initialization (v7.3.0+)
// Progressive Initialization
// =============================================
/**
@ -400,7 +399,6 @@ export class GcsStorage extends BaseStorage {
*
* @protected
* @override
* @since v7.3.0
*/
protected async runBackgroundInit(): Promise<void> {
const startTime = Date.now()
@ -423,7 +421,6 @@ export class GcsStorage extends BaseStorage {
* Stores result in bucketValidated/bucketValidationError for lazy use.
*
* @private
* @since v7.3.0
*/
private async validateBucketInBackground(): Promise<void> {
try {
@ -451,7 +448,6 @@ export class GcsStorage extends BaseStorage {
* Uses the existing initializeCounts() logic but in background.
*
* @private
* @since v7.3.0
*/
private async loadCountsInBackground(): Promise<void> {
try {
@ -475,7 +471,6 @@ export class GcsStorage extends BaseStorage {
* @throws Error if bucket does not exist or is not accessible
* @protected
* @override
* @since v7.3.0
*/
protected async ensureValidatedForWrite(): Promise<void> {
// If already validated, nothing to do
@ -562,7 +557,7 @@ export class GcsStorage extends BaseStorage {
}
/**
* Override base class to enable smart batching for cloud storage (v3.32.3+)
* Override base class to enable smart batching for cloud storage
*
* GCS is cloud storage with network latency (~50ms per write).
* Smart batching reduces writes from 1000 ops → 100 batches.
@ -596,7 +591,7 @@ export class GcsStorage extends BaseStorage {
}
}
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
/**
* Flush noun buffer to GCS
@ -628,20 +623,20 @@ export class GcsStorage extends BaseStorage {
await Promise.all(writes)
}
// v5.4.0: Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed saveNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Save a node to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.nounWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
@ -657,10 +652,10 @@ export class GcsStorage extends BaseStorage {
/**
* Save a node directly to GCS (bypass buffer)
*
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
* Performs lazy bucket validation on first write in progressive mode.
*/
private async saveNodeDirect(node: HNSWNode): Promise<void> {
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
// Apply backpressure before starting operation
@ -695,14 +690,14 @@ export class GcsStorage extends BaseStorage {
resumable: false // For small objects, non-resumable is faster
})
// CRITICAL FIX (v3.37.8): Only cache nodes with non-empty vectors
// CRITICAL FIX: Only cache nodes with non-empty vectors
// This prevents cache pollution from HNSW's lazy-loading nodes (vector: [])
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
// Note: Empty vectors are intentional during HNSW lazy mode - not logged
// Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveNounMetadata_internal
// This fixes the race condition where metadata didn't exist yet
this.logger.trace(`Node ${node.id} saved successfully`)
@ -721,7 +716,7 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed getNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Get a node from storage
@ -732,7 +727,7 @@ export class GcsStorage extends BaseStorage {
// Check cache first
const cached: HNSWNode | null = await this.nounCacheManager.get(id)
// Validate cached object before returning (v3.37.8+)
// Validate cached object before returning
if (cached !== undefined && cached !== null) {
// Validate cached object has required fields (including non-empty vector!)
if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
@ -831,18 +826,18 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
// Removed deleteNoun_internal - now inherit from BaseStorage's type-first implementation
/**
* Write an object to a specific path in GCS
* Primitive operation required by base class
*
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
* Performs lazy bucket validation on first write in progressive mode.
* @protected
*/
protected async writeObjectToPath(path: string, data: any): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -892,7 +887,7 @@ export class GcsStorage extends BaseStorage {
}
/**
* Batch read multiple objects from GCS (v5.12.0 - Cloud Storage Optimization)
* Batch read multiple objects from GCS
*
* **Performance**: GCS-optimized parallel downloads
* - Uses Promise.all() for concurrent requests
@ -908,7 +903,6 @@ export class GcsStorage extends BaseStorage {
* @returns Map of path → data (only successful reads included)
*
* @public - Called by baseStorage.readBatchFromAdapter()
* @since v5.12.0
*/
public async readBatch(paths: string[]): Promise<Map<string, any>> {
await this.ensureInitialized()
@ -960,12 +954,11 @@ export class GcsStorage extends BaseStorage {
}
/**
* Get GCS-specific batch configuration (v5.12.0)
* Get GCS-specific batch configuration
*
* GCS performs well with high concurrency due to HTTP/2 multiplexing
*
* @public - Overrides BaseStorage.getBatchConfig()
* @since v5.12.0
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -984,12 +977,12 @@ export class GcsStorage extends BaseStorage {
* Delete an object from a specific path in GCS
* Primitive operation required by base class
*
* v7.3.0: Performs lazy bucket validation on first delete in progressive mode.
* Performs lazy bucket validation on first delete in progressive mode.
* @protected
*/
protected async deleteObjectFromPath(path: string): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -1034,20 +1027,20 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed saveVerb_internal - now inherit from BaseStorage's type-first implementation
/**
* Save an edge to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.verbWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding verb ${edge.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
this.verbCacheManager.set(edge.id, edge)
await this.verbWriteBuffer.add(edge.id, edge)
@ -1061,10 +1054,10 @@ export class GcsStorage extends BaseStorage {
/**
* Save an edge directly to GCS (bypass buffer)
*
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
* Performs lazy bucket validation on first write in progressive mode.
*/
private async saveEdgeDirect(edge: Edge): Promise<void> {
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
const requestId = await this.applyBackpressure()
@ -1073,7 +1066,7 @@ export class GcsStorage extends BaseStorage {
this.logger.trace(`Saving edge ${edge.id}`)
// Convert connections Map to serializable format
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
@ -1085,7 +1078,7 @@ export class GcsStorage extends BaseStorage {
])
),
// CORE RELATIONAL DATA (v3.50.1+)
// CORE RELATIONAL DATA
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
@ -1107,7 +1100,7 @@ export class GcsStorage extends BaseStorage {
// Update cache
this.verbCacheManager.set(edge.id, edge)
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveVerbMetadata_internal
// This fixes the race condition where metadata didn't exist yet
this.logger.trace(`Edge ${edge.id} saved successfully`)
@ -1125,7 +1118,7 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed getVerb_internal - now inherit from BaseStorage's type-first implementation
/**
* Get an edge from storage
@ -1161,7 +1154,7 @@ export class GcsStorage extends BaseStorage {
connections.set(Number(level), new Set(verbIds as string[]))
}
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
// Return HNSWVerb with core relational fields (NO metadata field)
const edge: Edge = {
id: data.id,
vector: data.vector,
@ -1172,7 +1165,7 @@ export class GcsStorage extends BaseStorage {
sourceId: data.sourceId,
targetId: data.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
}
@ -1201,13 +1194,13 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.4.0: Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
// Removed deleteVerb_internal - now inherit from BaseStorage's type-first implementation
// v5.4.0: Removed pagination overrides - use BaseStorage's type-first implementation
// Removed pagination overrides - use BaseStorage's type-first implementation
// - getNounsWithPagination, getNodesWithPagination, getVerbsWithPagination
// - getNouns, getVerbs (public wrappers)
// v5.4.0: Removed 4 query *_internal methods - now inherit from BaseStorage's type-first implementation
// Removed 4 query *_internal methods - now inherit from BaseStorage's type-first implementation
// (getNounsByNounType_internal, getVerbsBySource_internal, getVerbsByTarget_internal, getVerbsByType_internal)
/**
@ -1280,7 +1273,7 @@ export class GcsStorage extends BaseStorage {
}
}
// v5.11.0: Clear ALL data using correct paths
// Clear ALL data using correct paths
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
await deleteObjectsWithPrefix('branches/')
@ -1290,7 +1283,7 @@ export class GcsStorage extends BaseStorage {
// Delete system metadata
await deleteObjectsWithPrefix('_system/')
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -1352,12 +1345,12 @@ export class GcsStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker object exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -1413,7 +1406,7 @@ export class GcsStorage extends BaseStorage {
}
} catch (error: any) {
if (error.code === 404) {
// CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart)
// CRITICAL FIX: Statistics file doesn't exist yet (first restart)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
this.logger.trace('Statistics file not found - returning minimal stats with counts')
@ -1607,11 +1600,11 @@ export class GcsStorage extends BaseStorage {
}
}
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
/**
* Get a noun's vector for HNSW rebuild
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
@ -1621,7 +1614,7 @@ export class GcsStorage extends BaseStorage {
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Uses mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -1630,7 +1623,7 @@ export class GcsStorage extends BaseStorage {
}): Promise<void> {
const lockKey = `hnsw/${nounId}`
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
// Problem: Without mutex, concurrent operations can:
// 1. Thread A reads noun (connections: [1,2,3])
// 2. Thread B reads noun (connections: [1,2,3])
@ -1650,7 +1643,7 @@ export class GcsStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
// Read existing noun data (if exists)
const existingNoun = await this.getNoun(nounId)
@ -1672,7 +1665,7 @@ export class GcsStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
await this.saveNoun(updatedNoun)
} finally {
// Release lock (ALWAYS runs, even if error thrown)
@ -1683,7 +1676,7 @@ export class GcsStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -1713,7 +1706,7 @@ export class GcsStorage extends BaseStorage {
* Save HNSW system data (entry point, max level)
* Storage path: system/hnsw-system.json
*
* CRITICAL FIX (v4.10.1): Optimistic locking with generation numbers to prevent race conditions
* CRITICAL FIX: Optimistic locking with generation numbers to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
@ -1807,7 +1800,7 @@ export class GcsStorage extends BaseStorage {
}
// ============================================================================
// GCS Lifecycle Management & Autoclass (v4.0.0)
// GCS Lifecycle Management & Autoclass
// Cost optimization through automatic tier transitions and Autoclass
// ============================================================================

10
src/storage/adapters/historicalStorageAdapter.ts
View file

@ -25,7 +25,7 @@
* - Lazy loading: only loads entities when accessed
* - No eager-loading of entire commit state
*
* v5.4.0: Production-ready, billion-scale historical queries
* Production-ready, billion-scale historical queries
*/
import { BaseStorage } from '../baseStorage.js'
@ -144,7 +144,7 @@ export class HistoricalStorageAdapter extends BaseStorage {
*/
public async init(): Promise<void> {
// Get COW components from underlying storage
// v6.2.4: Fixed property names - use public properties without underscore prefix
// Fixed property names - use public properties without underscore prefix
this.commitLog = this.underlyingStorage.commitLog
this.blobStorage = this.underlyingStorage.blobStorage
@ -161,7 +161,7 @@ export class HistoricalStorageAdapter extends BaseStorage {
throw new Error(`Commit not found: ${this.commitId}`)
}
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
}
@ -310,12 +310,12 @@ export class HistoricalStorageAdapter extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
* No-op for HistoricalStorageAdapter (read-only, doesn't manage COW)
* @returns Always false (read-only adapter doesn't manage COW state)
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/

49
src/storage/adapters/memoryStorage.ts
View file

@ -24,7 +24,7 @@ import { PaginatedResult } from '../../types/paginationTypes.js'
* Uses Maps to store data in memory
*/
export class MemoryStorage extends BaseStorage {
// v5.4.0: Removed redundant Maps (nouns, verbs) - objectStore handles all storage via type-first paths
// Removed redundant Maps (nouns, verbs) - objectStore handles all storage via type-first paths
private statistics: StatisticsData | null = null
// Unified object store for primitive operations (replaces metadata, nounMetadata, verbMetadata)
@ -55,7 +55,6 @@ export class MemoryStorage extends BaseStorage {
* - Parallel processing maximizes throughput
*
* @returns Memory-optimized batch configuration
* @since v4.11.0
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -72,27 +71,27 @@ export class MemoryStorage extends BaseStorage {
/**
* Initialize the storage adapter
* v6.0.0: Calls super.init() to initialize GraphAdjacencyIndex and type statistics
* Calls super.init() to initialize GraphAdjacencyIndex and type statistics
*/
public async init(): Promise<void> {
await super.init()
}
// v5.4.0: Removed saveNoun_internal and getNoun_internal - using BaseStorage's type-first implementation
// Removed saveNoun_internal and getNoun_internal - using BaseStorage's type-first implementation
/**
* Get nouns with pagination and filtering
* v4.0.0: Returns HNSWNounWithMetadata[] (includes metadata field)
* Returns HNSWNounWithMetadata[] (includes metadata field)
* @param options Pagination and filtering options
* @returns Promise that resolves to a paginated result of nouns with metadata
*/
// v5.4.0: Removed public method overrides (getNouns, getNounsWithPagination, getVerbs) - using BaseStorage's type-first implementation
// Removed public method overrides (getNouns, getNounsWithPagination, getVerbs) - using BaseStorage's type-first implementation
// v5.4.0: Removed getNounsByNounType_internal and deleteNoun_internal - using BaseStorage's type-first implementation
// Removed getNounsByNounType_internal and deleteNoun_internal - using BaseStorage's type-first implementation
// v5.4.0: Removed saveVerb_internal and getVerb_internal - using BaseStorage's type-first implementation
// Removed saveVerb_internal and getVerb_internal - using BaseStorage's type-first implementation
// v5.4.0: Removed verb *_internal method overrides - using BaseStorage's type-first implementation
// Removed verb *_internal method overrides - using BaseStorage's type-first implementation
/**
* Primitive operation: Write object to path
@ -159,8 +158,8 @@ export class MemoryStorage extends BaseStorage {
/**
* Clear all data from storage
* v5.4.0: Clears objectStore (type-first paths)
* v7.3.1: Also clears writeCache to prevent stale data after clear
* Clears objectStore (type-first paths)
* Also clears writeCache to prevent stale data after clear
*/
public async clear(): Promise<void> {
this.objectStore.clear()
@ -174,7 +173,7 @@ export class MemoryStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// v7.3.1: Clear write-through cache (inherited from BaseStorage)
// Clear write-through cache (inherited from BaseStorage)
// Without this, readWithInheritance() would return stale cached data
// after clear(), causing "ghost" entities to appear
this.clearWriteCache()
@ -182,7 +181,7 @@ export class MemoryStorage extends BaseStorage {
/**
* Get information about storage usage and capacity
* v5.4.0: Uses BaseStorage counts
* Uses BaseStorage counts
*/
public async getStorageStatus(): Promise<{
type: string
@ -204,12 +203,12 @@ export class MemoryStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: No-op for MemoryStorage (doesn't persist)
* No-op for MemoryStorage (doesn't persist)
* @returns Always false (marker doesn't persist in memory)
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -252,7 +251,7 @@ export class MemoryStorage extends BaseStorage {
*/
protected async getStatisticsData(): Promise<StatisticsData | null> {
if (!this.statistics) {
// CRITICAL FIX (v3.37.4): Statistics don't exist yet (first init)
// CRITICAL FIX: Statistics don't exist yet (first init)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
@ -299,10 +298,10 @@ export class MemoryStorage extends BaseStorage {
}
/**
* Initialize counts from in-memory storage - O(1) operation (v4.0.0)
* Initialize counts from in-memory storage - O(1) operation
*/
protected async initializeCounts(): Promise<void> {
// v6.0.0: Scan objectStore paths (ID-first structure) to count entities
// Scan objectStore paths (ID-first structure) to count entities
this.entityCounts.clear()
this.verbCounts.clear()
@ -314,14 +313,14 @@ export class MemoryStorage extends BaseStorage {
// Count nouns (entities/nouns/{shard}/{id}/vectors.json)
const nounMatch = path.match(/^entities\/nouns\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/)
if (nounMatch) {
// v6.0.0: Type is in metadata, not path - just count total
// Type is in metadata, not path - just count total
totalNouns++
}
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
const verbMatch = path.match(/^entities\/verbs\/[0-9a-f]{2}\/[^/]+\/vectors\.json$/)
if (verbMatch) {
// v6.0.0: Type is in metadata, not path - just count total
// Type is in metadata, not path - just count total
totalVerbs++
}
}
@ -339,26 +338,26 @@ export class MemoryStorage extends BaseStorage {
}
// =============================================
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
// =============================================
/**
* Get vector for a noun
* v5.4.0: Uses BaseStorage's type-first implementation
* Uses BaseStorage's type-first implementation
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
return noun ? [...noun.vector] : null
}
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
// CRITICAL FIX: Mutex locks for HNSW concurrency control
// Even in-memory operations need serialization to prevent async race conditions
private hnswLocks = new Map<string, Promise<void>>()
/**
* Save HNSW graph data for a noun
*
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions during concurrent HNSW updates
* CRITICAL FIX: Mutex locking to prevent race conditions during concurrent HNSW updates
* Even in-memory operations can race due to async/await interleaving
* Prevents data corruption when multiple entities connect to same neighbor simultaneously
*/
@ -417,7 +416,7 @@ export class MemoryStorage extends BaseStorage {
/**
* Save HNSW system data (entry point, max level)
*
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions
* CRITICAL FIX: Mutex locking to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null

43
src/storage/adapters/opfsStorage.ts
View file

@ -57,7 +57,7 @@ const ROOT_DIR = 'opfs-vector-db'
* OPFS storage adapter for browser environments
* Uses the Origin Private File System API to store data persistently
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -97,7 +97,6 @@ export class OPFSStorage extends BaseStorage {
* - Sequential processing preferred for stability
*
* @returns OPFS-optimized batch configuration
* @since v4.11.0
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -172,7 +171,7 @@ export class OPFSStorage extends BaseStorage {
// Initialize counts from storage
await this.initializeCounts()
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
} catch (error) {
console.error('Failed to initialize OPFS storage:', error)
@ -232,7 +231,7 @@ export class OPFSStorage extends BaseStorage {
}
}
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
/**
* Delete an edge from storage
@ -482,14 +481,14 @@ export class OPFSStorage extends BaseStorage {
// Remove all files in the index directory
await removeDirectoryContents(this.indexDir!)
// v5.6.1: Remove COW (copy-on-write) version control data
// Remove COW (copy-on-write) version control data
// This directory stores all git-like versioning data (commits, trees, blobs, refs)
// Must be deleted to fully clear all data including version history
try {
// Delete the entire _cow/ directory (not just contents)
await this.rootDir!.removeEntry('_cow', { recursive: true })
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -505,7 +504,7 @@ export class OPFSStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
// Reset entity counters (inherited from BaseStorageAdapter)
// These in-memory counters must be reset to 0 after clearing all data
;(this as any).totalNounCount = 0
;(this as any).totalVerbCount = 0
@ -517,16 +516,16 @@ export class OPFSStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker file exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
// Quota monitoring configuration (v4.0.0)
// Quota monitoring configuration
private quotaWarningThreshold = 0.8 // Warn at 80% usage
private quotaCriticalThreshold = 0.95 // Critical at 95% usage
private lastQuotaCheck: number = 0
@ -675,7 +674,7 @@ export class OPFSStorage extends BaseStorage {
}
/**
* Get detailed quota status with warnings (v4.0.0)
* Get detailed quota status with warnings
* Monitors storage usage and warns when approaching quota limits
*
* @returns Promise that resolves to quota status with warning levels
@ -769,7 +768,7 @@ export class OPFSStorage extends BaseStorage {
}
/**
* Monitor quota during operations (v4.0.0)
* Monitor quota during operations
* Automatically checks quota at regular intervals and warns if approaching limits
* Call this before write operations to ensure quota is available
*
@ -1185,7 +1184,7 @@ export class OPFSStorage extends BaseStorage {
}
}
} catch (error) {
// CRITICAL FIX (v3.37.4): No statistics files exist (first init)
// CRITICAL FIX: No statistics files exist (first init)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
@ -1228,7 +1227,7 @@ export class OPFSStorage extends BaseStorage {
* @param options Pagination and filter options
* @returns Promise that resolves to a paginated result of nouns
*/
// v5.4.0: Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation
// Removed pagination overrides (getNounsWithPagination, getVerbsWithPagination) - use BaseStorage's type-first implementation
/**
* Initialize counts from OPFS storage
@ -1313,28 +1312,28 @@ export class OPFSStorage extends BaseStorage {
}
}
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
/**
* Get a noun's vector for HNSW rebuild
*/
/**
* Get vector for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
return noun ? noun.vector : null
}
// CRITICAL FIX (v4.10.1): Mutex locks for HNSW concurrency control
// CRITICAL FIX: Mutex locks for HNSW concurrency control
// Browser environments are single-threaded but async operations can still interleave
private hnswLocks = new Map<string, Promise<void>>()
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Preserves mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -1354,7 +1353,7 @@ export class OPFSStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
const existingNoun = await this.getNoun(nounId)
if (!existingNoun) {
@ -1374,7 +1373,7 @@ export class OPFSStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths)
// Use BaseStorage's saveNoun (type-first paths)
await this.saveNoun(updatedNoun)
} finally {
// Release lock
@ -1385,7 +1384,7 @@ export class OPFSStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -1415,7 +1414,7 @@ export class OPFSStorage extends BaseStorage {
* Save HNSW system data (entry point, max level)
* Storage path: index/hnsw-system.json
*
* CRITICAL FIX (v4.10.1): Mutex locking to prevent race conditions
* CRITICAL FIX: Mutex locking to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null

4
src/storage/adapters/optimizedS3Search.ts
View file

@ -107,7 +107,7 @@ export class OptimizedS3Search {
}
// Determine if there are more items
const hasMore = listResult.hasMore || nouns.length > limit // v5.7.11: Fixed >= to > (was causing infinite loop)
const hasMore = listResult.hasMore || nouns.length > limit // Fixed >= to > (was causing infinite loop)
// Set next cursor
let nextCursor: string | undefined
@ -190,7 +190,7 @@ export class OptimizedS3Search {
}
// Determine if there are more items
const hasMore = listResult.hasMore || verbs.length > limit // v5.7.11: Fixed >= to > (was causing infinite loop)
const hasMore = listResult.hasMore || verbs.length > limit // Fixed >= to > (was causing infinite loop)
// Set next cursor
let nextCursor: string | undefined

49
src/storage/adapters/r2Storage.ts
View file

@ -58,7 +58,7 @@ const MAX_R2_PAGE_SIZE = 1000
* Dedicated Cloudflare R2 storage adapter
* Optimized for R2's unique characteristics and global edge network
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed getNounsWithPagination override
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -112,7 +112,7 @@ export class R2Storage extends BaseStorage {
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
// v6.2.7: Write buffering always enabled for consistent performance
// Write buffering always enabled for consistent performance
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
// Multi-level cache manager for efficient data access
@ -122,7 +122,7 @@ export class R2Storage extends BaseStorage {
// Module logger
private logger = createModuleLogger('R2Storage')
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
// HNSW mutex locks to prevent read-modify-write races
private hnswLocks = new Map<string, Promise<void>>()
/**
@ -168,7 +168,7 @@ export class R2Storage extends BaseStorage {
})
this.verbCacheManager = new CacheManager<Edge>(options.cacheConfig)
// v6.2.7: Write buffering always enabled - no env var check needed
// Write buffering always enabled - no env var check needed
}
/**
@ -183,7 +183,7 @@ export class R2Storage extends BaseStorage {
* Zero egress fees enable aggressive caching and parallel downloads
*
* @returns R2-optimized batch configuration
* @since v5.12.0 - Updated for native batch API
* Updated for native batch API
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -208,7 +208,6 @@ export class R2Storage extends BaseStorage {
*
* @param paths - Array of R2 object keys to read
* @returns Map of path -> parsed JSON data (only successful reads)
* @since v5.12.0
*/
public async readBatch(paths: string[]): Promise<Map<string, any>> {
await this.ensureInitialized()
@ -336,7 +335,7 @@ export class R2Storage extends BaseStorage {
this.nounCacheManager.clear()
this.verbCacheManager.clear()
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
} catch (error) {
this.logger.error('Failed to initialize R2 storage:', error)
@ -409,7 +408,7 @@ export class R2Storage extends BaseStorage {
}
}
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
/**
* Flush noun buffer to R2
@ -444,16 +443,16 @@ export class R2Storage extends BaseStorage {
/**
* Save a node to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.nounWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
@ -728,14 +727,14 @@ export class R2Storage extends BaseStorage {
/**
* Save an edge to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveEdge(edge: Edge): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.verbWriteBuffer) {
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
this.verbCacheManager.set(edge.id, edge)
await this.verbWriteBuffer.add(edge.id, edge)
@ -750,7 +749,7 @@ export class R2Storage extends BaseStorage {
const requestId = await this.applyBackpressure()
try {
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
// ARCHITECTURAL FIX: Include core relational fields in verb vector file
// These fields are essential for 90% of operations - no metadata lookup needed
const serializableEdge = {
id: edge.id,
@ -762,7 +761,7 @@ export class R2Storage extends BaseStorage {
])
),
// CORE RELATIONAL DATA (v3.50.1+)
// CORE RELATIONAL DATA
verb: edge.verb,
sourceId: edge.sourceId,
targetId: edge.targetId,
@ -785,7 +784,7 @@ export class R2Storage extends BaseStorage {
this.verbCacheManager.set(edge.id, edge)
// Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2)
// Count tracking happens in baseStorage.saveVerbMetadata_internal
// This fixes the race condition where metadata didn't exist yet
this.releaseBackpressure(true, requestId)
@ -831,7 +830,7 @@ export class R2Storage extends BaseStorage {
connections.set(Number(level), new Set(verbIds as string[]))
}
// v4.0.0: Return HNSWVerb with core relational fields (NO metadata field)
// Return HNSWVerb with core relational fields (NO metadata field)
const edge: Edge = {
id: data.id,
vector: data.vector,
@ -842,7 +841,7 @@ export class R2Storage extends BaseStorage {
sourceId: data.sourceId,
targetId: data.targetId
// ✅ NO metadata field in v4.0.0
// ✅ NO metadata field
// User metadata retrieved separately via getVerbMetadata()
}
@ -1081,7 +1080,7 @@ export class R2Storage extends BaseStorage {
prodLog.info('🧹 R2: Clearing all data from bucket...')
// v5.11.0: Clear ALL data using correct paths
// Clear ALL data using correct paths
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
const branchObjects = await this.listObjectsUnderPath('branches/')
for (const key of branchObjects) {
@ -1100,7 +1099,7 @@ export class R2Storage extends BaseStorage {
await this.deleteObjectFromPath(key)
}
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -1143,17 +1142,17 @@ export class R2Storage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker object exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
// Removed getNounsWithPagination override - use BaseStorage's type-first implementation
// v5.4.0: Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
// Removed 10 *_internal method overrides - now inherit from BaseStorage's type-first implementation
}

98
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -85,7 +85,7 @@ type S3Command = any
* - endpoint: GCS endpoint (e.g., 'https://storage.googleapis.com')
* - bucketName: GCS bucket name
*
* v5.4.0: Type-aware storage now built into BaseStorage
* Type-aware storage now built into BaseStorage
* - Removed 10 *_internal method overrides (now inherit from BaseStorage's type-first implementation)
* - Removed 2 pagination method overrides (getNounsWithPagination, getVerbsWithPagination)
* - Updated HNSW methods to use BaseStorage's getNoun/saveNoun (type-first paths)
@ -152,7 +152,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Request coalescer for deduplication
private requestCoalescer: RequestCoalescer | null = null
// v6.2.7: Write buffering always enabled for consistent performance
// Write buffering always enabled for consistent performance
// Removes dynamic mode switching complexity - cloud storage always benefits from batching
// Operation executors for timeout and retry handling
@ -165,7 +165,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Module logger
private logger = createModuleLogger('S3Storage')
// v5.4.0: HNSW mutex locks to prevent read-modify-write races
// HNSW mutex locks to prevent read-modify-write races
private hnswLocks = new Map<string, Promise<void>>()
/**
@ -210,7 +210,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Lambda, Cloud Run),
* strict locally. Zero-config optimization.
@ -219,7 +219,6 @@ export class S3CompatibleStorage extends BaseStorage {
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: InitMode
@ -236,7 +235,7 @@ export class S3CompatibleStorage extends BaseStorage {
this.serviceType = options.serviceType || 's3'
this.readOnly = options.readOnly || false
// v7.3.0: Handle initMode
// Handle initMode
if (options.initMode) {
this.initMode = options.initMode
}
@ -270,7 +269,7 @@ export class S3CompatibleStorage extends BaseStorage {
* S3 supports ~5000 operations/second with burst capacity up to 10,000
*
* @returns S3-optimized batch configuration
* @since v5.12.0 - Updated for native batch API
* Updated for native batch API
*/
public getBatchConfig(): StorageBatchConfig {
return {
@ -295,7 +294,6 @@ export class S3CompatibleStorage extends BaseStorage {
*
* @param paths - Array of S3 object keys to read
* @returns Map of path -> parsed JSON data (only successful reads)
* @since v5.12.0
*/
public async readBatch(paths: string[]): Promise<Map<string, any>> {
await this.ensureInitialized()
@ -358,7 +356,7 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Initialize the storage adapter
*
* v7.3.0: Supports progressive initialization for fast cold starts
* Supports progressive initialization for fast cold starts
*
* | Mode | Init Time | When |
* |------|-----------|------|
@ -514,7 +512,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Initialize request coalescer
this.initializeCoalescer()
// CRITICAL FIX (v3.37.7): Clear any stale cache entries from previous runs
// CRITICAL FIX: Clear any stale cache entries from previous runs
// This prevents cache poisoning from causing silent failures on container restart
const nodeCacheSize = this.nodeCache?.size || 0
if (nodeCacheSize > 0) {
@ -524,7 +522,7 @@ export class S3CompatibleStorage extends BaseStorage {
prodLog.info('🧹 Node cache is empty - starting fresh')
}
// v7.3.0: Progressive vs Strict initialization
// Progressive vs Strict initialization
if (effectiveMode === 'progressive') {
// PROGRESSIVE MODE: Fast init, background validation
// Mark as initialized immediately - ready to accept operations
@ -533,7 +531,7 @@ export class S3CompatibleStorage extends BaseStorage {
prodLog.info(`✅ S3 progressive init complete: ${this.bucketName} (validation deferred)`)
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Schedule background tasks (non-blocking)
@ -556,7 +554,7 @@ export class S3CompatibleStorage extends BaseStorage {
await this.initializeCounts()
this.countsLoaded = true
// v6.0.0: Initialize GraphAdjacencyIndex and type statistics
// Initialize GraphAdjacencyIndex and type statistics
await super.init()
// Mark background tasks as complete (nothing to do in background)
@ -573,7 +571,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
// =============================================
// Progressive Initialization (v7.3.0+)
// Progressive Initialization
// =============================================
/**
@ -588,7 +586,6 @@ export class S3CompatibleStorage extends BaseStorage {
*
* @protected
* @override
* @since v7.3.0
*/
protected async runBackgroundInit(): Promise<void> {
const startTime = Date.now()
@ -614,7 +611,6 @@ export class S3CompatibleStorage extends BaseStorage {
* Stores result in bucketValidated/bucketValidationError for lazy use.
*
* @private
* @since v7.3.0
*/
private async validateBucketInBackground(): Promise<void> {
try {
@ -638,7 +634,6 @@ export class S3CompatibleStorage extends BaseStorage {
* Load counts from storage in background.
*
* @private
* @since v7.3.0
*/
private async loadCountsInBackground(): Promise<void> {
try {
@ -662,7 +657,6 @@ export class S3CompatibleStorage extends BaseStorage {
* @throws Error if bucket does not exist or is not accessible
* @protected
* @override
* @since v7.3.0
*/
protected async ensureValidatedForWrite(): Promise<void> {
// If already validated, nothing to do
@ -918,7 +912,7 @@ export class S3CompatibleStorage extends BaseStorage {
)
}
// v6.2.7: Removed checkVolumeMode() - write buffering always enabled for cloud storage
// Removed checkVolumeMode() - write buffering always enabled for cloud storage
/**
* Bulk write nouns to S3
@ -1179,20 +1173,20 @@ export class S3CompatibleStorage extends BaseStorage {
return this.socketManager.getBatchSize()
}
// v5.4.0: Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation
// Removed 10 *_internal method overrides (lines 984-2069) - now inherit from BaseStorage's type-first implementation
/**
* Save a node to storage
* v6.2.7: Always uses write buffer for consistent performance
* Always uses write buffer for consistent performance
*/
protected async saveNode(node: HNSWNode): Promise<void> {
await this.ensureInitialized()
// v6.2.7: Always use write buffer - cloud storage benefits from batching
// Always use write buffer - cloud storage benefits from batching
if (this.nounWriteBuffer) {
this.logger.trace(`📝 BUFFERING: Adding noun ${node.id} to write buffer`)
// v6.2.6: Populate cache BEFORE buffering for read-after-write consistency
// Populate cache BEFORE buffering for read-after-write consistency
if (node.vector && Array.isArray(node.vector) && node.vector.length > 0) {
this.nounCacheManager.set(node.id, node)
}
@ -1242,7 +1236,7 @@ export class S3CompatibleStorage extends BaseStorage {
this.logger.debug(`Node ${node.id} saved successfully`)
// Log the change for efficient synchronization (v4.0.0: no metadata on node)
// Log the change for efficient synchronization (no metadata on node)
await this.appendToChangeLog({
timestamp: Date.now(),
operation: 'add', // Could be 'update' if we track existing nodes
@ -1250,7 +1244,7 @@ export class S3CompatibleStorage extends BaseStorage {
entityId: node.id,
data: {
vector: node.vector
// ✅ NO metadata field in v4.0.0 - stored separately
// ✅ NO metadata field - stored separately
}
})
@ -1296,7 +1290,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
// v5.4.0: Removed getNoun_internal override - uses BaseStorage type-first implementation
// Removed getNoun_internal override - uses BaseStorage type-first implementation
/**
* Get a node from storage
@ -1307,7 +1301,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Check cache first
const cached = this.nodeCache.get(id)
// Validate cached object before returning (v3.37.8+)
// Validate cached object before returning
if (cached !== undefined && cached !== null) {
// Validate cached object has required fields (including non-empty vector!)
if (!cached.id || !cached.vector || !Array.isArray(cached.vector) || cached.vector.length === 0) {
@ -1607,7 +1601,7 @@ export class S3CompatibleStorage extends BaseStorage {
return nodes
}
// v5.4.0: Removed 4 *_internal method overrides (getNounsByNounType_internal, deleteNoun_internal, saveVerb_internal, getVerb_internal)
// Removed 4 *_internal method overrides (getNounsByNounType_internal, deleteNoun_internal, saveVerb_internal, getVerb_internal)
// Now inherit from BaseStorage's type-first implementation
@ -1788,23 +1782,23 @@ export class S3CompatibleStorage extends BaseStorage {
return true // Return all edges since filtering requires metadata
}
// v5.4.0: Removed getVerbsWithPagination override - use BaseStorage's type-first implementation
// Removed getVerbsWithPagination override - use BaseStorage's type-first implementation
// v5.4.0: Removed 4 more *_internal method overrides (getVerbsBySource, getVerbsByTarget, getVerbsByType, deleteVerb)
// Removed 4 more *_internal method overrides (getVerbsBySource, getVerbsByTarget, getVerbsByType, deleteVerb)
// Total: 8 *_internal methods removed - all now inherit from BaseStorage's type-first implementation
/**
* Primitive operation: Write object to path
* All metadata operations use this internally via base class routing
*
* v7.3.0: Performs lazy bucket validation on first write in progressive mode.
* Performs lazy bucket validation on first write in progressive mode.
*/
protected async writeObjectToPath(path: string, data: any): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
// Apply backpressure before starting operation
@ -1897,11 +1891,11 @@ export class S3CompatibleStorage extends BaseStorage {
* Primitive operation: Delete object from path
* All metadata operations use this internally via base class routing
*
* v7.3.0: Performs lazy bucket validation on first delete in progressive mode.
* Performs lazy bucket validation on first delete in progressive mode.
*/
protected async deleteObjectFromPath(path: string): Promise<void> {
await this.ensureInitialized()
// v7.3.0: Lazy bucket validation for progressive init
// Lazy bucket validation for progressive init
await this.ensureValidatedForWrite()
try {
@ -2315,7 +2309,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
// v5.11.0: Clear ALL data using correct paths
// Clear ALL data using correct paths
// Delete entire branches/ directory (includes ALL entities, ALL types, ALL VFS data, ALL forks)
await deleteObjectsWithPrefix('branches/')
@ -2325,7 +2319,7 @@ export class S3CompatibleStorage extends BaseStorage {
// Delete system metadata
await deleteObjectsWithPrefix('_system/')
// v5.11.0: Reset COW managers (but don't disable COW - it's always enabled)
// Reset COW managers (but don't disable COW - it's always enabled)
// COW will re-initialize automatically on next use
this.refManager = undefined
this.blobStorage = undefined
@ -2335,7 +2329,7 @@ export class S3CompatibleStorage extends BaseStorage {
this.statisticsCache = null
this.statisticsModified = false
// v5.6.1: Reset entity counters (inherited from BaseStorageAdapter)
// Reset entity counters (inherited from BaseStorageAdapter)
// These in-memory counters must be reset to 0 after clearing all data
;(this as any).totalNounCount = 0
;(this as any).totalVerbCount = 0
@ -2457,12 +2451,12 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Check if COW has been explicitly disabled via clear()
* v5.10.4: Fixes bug where clear() doesn't persist across instance restarts
* Fixes bug where clear() doesn't persist across instance restarts
* @returns true if marker object exists, false otherwise
* @protected
*/
/**
* v5.11.0: Removed checkClearMarker() and createClearMarker() methods
* Removed checkClearMarker() and createClearMarker() methods
* COW is now always enabled - marker files are no longer used
*/
@ -2851,7 +2845,7 @@ export class S3CompatibleStorage extends BaseStorage {
totalEdges: this.totalVerbCount
}
}
// CRITICAL FIX (v3.37.4): Statistics file doesn't exist yet (first restart)
// CRITICAL FIX: Statistics file doesn't exist yet (first restart)
// Return minimal stats with counts instead of null
// This prevents HNSW from seeing entityCount=0 during index rebuild
return {
@ -3415,7 +3409,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
}
// v5.4.0: Removed getNounsWithPagination override - use BaseStorage's type-first implementation
// Removed getNounsWithPagination override - use BaseStorage's type-first implementation
/**
* Estimate total noun count by listing objects across all shards
@ -3549,7 +3543,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Override base class to enable smart batching for cloud storage (v3.32.3+)
* Override base class to enable smart batching for cloud storage
*
* S3 is cloud storage with network latency (~50ms per write).
* Smart batching reduces writes from 1000 ops → 100 batches.
@ -3560,11 +3554,11 @@ export class S3CompatibleStorage extends BaseStorage {
return true // S3 benefits from batching
}
// HNSW Index Persistence (v3.35.0+)
// HNSW Index Persistence
/**
* Get a noun's vector for HNSW rebuild
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getNounVector(id: string): Promise<number[] | null> {
const noun = await this.getNoun(id)
@ -3574,7 +3568,7 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Save HNSW graph data for a noun
*
* v5.4.0: Uses BaseStorage's getNoun/saveNoun (type-first paths)
* Uses BaseStorage's getNoun/saveNoun (type-first paths)
* CRITICAL: Uses mutex locking to prevent read-modify-write races
*/
public async saveHNSWData(nounId: string, hnswData: {
@ -3583,7 +3577,7 @@ export class S3CompatibleStorage extends BaseStorage {
}): Promise<void> {
const lockKey = `hnsw/${nounId}`
// CRITICAL FIX (v4.10.1): Mutex lock to prevent read-modify-write races
// CRITICAL FIX: Mutex lock to prevent read-modify-write races
// Problem: Without mutex, concurrent operations can:
// 1. Thread A reads noun (connections: [1,2,3])
// 2. Thread B reads noun (connections: [1,2,3])
@ -3603,7 +3597,7 @@ export class S3CompatibleStorage extends BaseStorage {
this.hnswLocks.set(lockKey, lockPromise)
try {
// v5.4.0: Use BaseStorage's getNoun (type-first paths)
// Use BaseStorage's getNoun (type-first paths)
// Read existing noun data (if exists)
const existingNoun = await this.getNoun(nounId)
@ -3625,7 +3619,7 @@ export class S3CompatibleStorage extends BaseStorage {
connections: connectionsMap
}
// v5.4.0: Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
// Use BaseStorage's saveNoun (type-first paths, atomic write via writeObjectToBranch)
await this.saveNoun(updatedNoun)
} finally {
// Release lock (ALWAYS runs, even if error thrown)
@ -3636,7 +3630,7 @@ export class S3CompatibleStorage extends BaseStorage {
/**
* Get HNSW graph data for a noun
* v5.4.0: Uses BaseStorage's getNoun (type-first paths)
* Uses BaseStorage's getNoun (type-first paths)
*/
public async getHNSWData(nounId: string): Promise<{
level: number
@ -3666,7 +3660,7 @@ export class S3CompatibleStorage extends BaseStorage {
* Save HNSW system data (entry point, max level)
* Storage path: system/hnsw-system.json
*
* CRITICAL FIX (v4.10.1): Optimistic locking with ETags to prevent race conditions
* CRITICAL FIX: Optimistic locking with ETags to prevent race conditions
*/
public async saveHNSWSystem(systemData: {
entryPointId: string | null
@ -3781,7 +3775,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Set S3 lifecycle policy for automatic tier transitions and deletions (v4.0.0)
* Set S3 lifecycle policy for automatic tier transitions and deletions
* Automates cost optimization by moving old data to cheaper storage classes
*
* S3 Storage Classes:
@ -3976,7 +3970,7 @@ export class S3CompatibleStorage extends BaseStorage {
}
/**
* Enable S3 Intelligent-Tiering for automatic cost optimization (v4.0.0)
* Enable S3 Intelligent-Tiering for automatic cost optimization
* Automatically moves objects between access tiers based on usage patterns
*
* Intelligent-Tiering automatically saves up to 95% on storage costs:

4
src/storage/backwardCompatibility.ts
View file

@ -1,6 +1,6 @@
/**
* DEPRECATED (v4.7.2): Backward compatibility stubs
* TODO: Remove in v4.7.3 after migrating s3CompatibleStorage
* DEPRECATED: Backward compatibility stubs
* TODO: Remove after migrating s3CompatibleStorage
*/
export class StorageCompatibilityLayer {

339
src/storage/baseStorage.ts
View file

File diff suppressed because it is too large Load diff

14
src/storage/cow/BlobStorage.ts
View file

@ -161,7 +161,7 @@ export class BlobStorage {
}
/**
* v5.7.5: Ensure compression is ready before write operations
* Ensure compression is ready before write operations
* Fixes race condition where write happens before async compression init completes
*/
private async ensureCompressionReady(): Promise<void> {
@ -206,7 +206,7 @@ export class BlobStorage {
return hash
}
// v5.7.5: Ensure compression is initialized before writing
// Ensure compression is initialized before writing
// Fixes race condition where write happens before async init completes
await this.ensureCompressionReady()
@ -223,7 +223,7 @@ export class BlobStorage {
}
// Create metadata
// v5.7.5: Store ACTUAL compression state, not intended
// Store ACTUAL compression state, not intended
// Prevents corruption if compression failed to initialize
const actualCompression = finalData === data ? 'none' : compression
const metadata: BlobMetadata = {
@ -277,7 +277,7 @@ export class BlobStorage {
* @returns Blob data
*/
async read(hash: string, options: BlobReadOptions = {}): Promise<Buffer> {
// v5.3.4 fix: Guard against NULL hash (sentinel value)
// Guard against NULL hash (sentinel value)
// NULL_HASH ('0000...0000') is used as a sentinel for "no parent" or "empty tree"
// It should NEVER be read as actual blob data
if (isNullHash(hash)) {
@ -309,7 +309,7 @@ export class BlobStorage {
metadataBuffer = await this.adapter.get(`${tryPrefix}-meta:${hash}`)
if (metadataBuffer) {
prefix = tryPrefix
// v5.10.1: Unwrap metadata before parsing (defense-in-depth)
// Unwrap metadata before parsing (defense-in-depth)
// Metadata should be JSON, but adapter might return wrapped format
const unwrappedMetadata = unwrapBinaryData(metadataBuffer)
metadata = JSON.parse(unwrappedMetadata.toString())
@ -338,8 +338,8 @@ export class BlobStorage {
finalData = await this.zstdDecompress(data)
}
// v5.10.1: Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression)
// Even though COW adapter should unwrap (v5.7.5), verify it happened and re-unwrap if needed
// Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression)
// Even though COW adapter should unwrap, verify it happened and re-unwrap if needed
// This prevents "Blob integrity check failed" errors if adapter returns wrapped data
// Uses shared binaryDataCodec utility (single source of truth for unwrap logic)
const unwrappedData = unwrapBinaryData(finalData)

2
src/storage/cow/CommitLog.ts
View file

@ -43,7 +43,7 @@ export interface CommitLogStats {
/**
* CommitLog: Efficient commit history traversal and querying
*
* Pure v5.0.0 implementation - modern, clean, fast
* Pure implementation - modern, clean, fast
*/
export class CommitLog {
private blobStorage: BlobStorage

2
src/storage/cow/CommitObject.ts
View file

@ -337,7 +337,7 @@ export class CommitObject {
let currentHash: string | null = startHash
let depth = 0
// v5.3.4 fix: Guard against NULL hash (sentinel for "no parent")
// Guard against NULL hash (sentinel for "no parent")
// The initial commit has parent = null or NULL_HASH ('0000...0000')
// We must stop walking when we reach it, not try to read it
while (currentHash && !isNullHash(currentHash)) {

2
src/storage/cow/RefManager.ts
View file

@ -55,7 +55,7 @@ export interface RefUpdateOptions {
/**
* RefManager: Manages branches, tags, and HEAD pointer
*
* Pure implementation for v5.0.0 - no backward compatibility
* Pure implementation - no backward compatibility
*/
export class RefManager {
private adapter: COWStorageAdapter

4
src/storage/cow/binaryDataCodec.ts
View file

@ -86,7 +86,7 @@ export function unwrapBinaryData(data: any): Buffer {
/**
* Wrap binary data for JSON storage
*
* ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH! (v6.2.0)
* ⚠️ WARNING: DO NOT USE THIS ON WRITE PATH!
* ⚠️ Use key-based dispatch in baseStorage.ts COW adapter instead.
* ⚠️ This function exists for legacy/compatibility only.
*
@ -94,7 +94,7 @@ export function unwrapBinaryData(data: any): Buffer {
* This is FRAGILE because compressed binary can accidentally parse as valid JSON,
* causing blob integrity failures.
*
* v6.2.0 SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
* SOLUTION: baseStorage.ts COW adapter now uses key naming convention:
* - Keys with '-meta:' or 'ref:' prefix → Always JSON
* - Keys with 'blob:', 'commit:', 'tree:' prefix → Always binary
* No guessing needed!

23
src/storage/storageFactory.ts
View file

@ -10,7 +10,7 @@ import { S3CompatibleStorage } from './adapters/s3CompatibleStorage.js'
import { R2Storage } from './adapters/r2Storage.js'
import { GcsStorage } from './adapters/gcsStorage.js'
import { AzureBlobStorage } from './adapters/azureBlobStorage.js'
// TypeAwareStorageAdapter removed in v5.4.0 - type-aware now built into BaseStorage
// TypeAwareStorageAdapter removed - type-aware now built into BaseStorage
// FileSystemStorage is dynamically imported to avoid issues in browser environments
import { isBrowser } from '../utils/environment.js'
import { OperationConfig } from '../utils/operationUtils.js'
@ -94,7 +94,7 @@ export interface StorageOptions {
sessionToken?: string
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Lambda),
* strict locally. Zero-config optimization.
@ -103,7 +103,6 @@ export interface StorageOptions {
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: 'progressive' | 'strict' | 'auto'
}
@ -215,7 +214,7 @@ export interface StorageOptions {
skipCountsFile?: boolean
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Cloud Run),
* strict locally. Zero-config optimization.
@ -224,7 +223,6 @@ export interface StorageOptions {
* - `'strict'`: Traditional blocking init. Validates bucket and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: 'progressive' | 'strict' | 'auto'
}
@ -259,7 +257,7 @@ export interface StorageOptions {
sasToken?: string
/**
* Initialization mode for fast cold starts (v7.3.0+)
* Initialization mode for fast cold starts
*
* - `'auto'` (default): Progressive in cloud environments (Azure Functions),
* strict locally. Zero-config optimization.
@ -268,7 +266,6 @@ export interface StorageOptions {
* - `'strict'`: Traditional blocking init. Validates container and loads counts
* before init() returns.
*
* @since v7.3.0
*/
initMode?: 'progressive' | 'strict' | 'auto'
}
@ -388,7 +385,7 @@ export interface StorageOptions {
/**
* COW (Copy-on-Write) configuration for instant fork() capability
* v5.0.1: COW is now always enabled (automatic, zero-config)
* COW is now always enabled (automatic, zero-config)
*/
branch?: string // Current branch name (default: 'main')
enableCompression?: boolean // Enable zstd compression for COW blobs (default: true)
@ -410,14 +407,14 @@ function getFileSystemPath(options: StorageOptions): string {
}
/**
* Configure COW (Copy-on-Write) options on a storage adapter (v5.4.0)
* Configure COW (Copy-on-Write) options on a storage adapter
* TypeAware is now built-in to all adapters, no wrapper needed!
*
* @param storage - The storage adapter
* @param options - Storage options (for COW configuration)
*/
function configureCOW(storage: any, options?: StorageOptions): void {
// v5.0.1: COW will be initialized AFTER storage.init() in Brainy
// COW will be initialized AFTER storage.init() in Brainy
// Store COW options for later initialization
if (typeof storage.initializeCOW === 'function') {
storage._cowOptions = {
@ -672,10 +669,10 @@ export async function createStorage(
}
case 'type-aware':
// v5.0.0: TypeAware is now the default for ALL adapters
// TypeAware is now the default for ALL adapters
// Redirect to the underlying type instead
console.warn(
'⚠️ type-aware is deprecated in v5.0.0 - TypeAware is now always enabled.'
'⚠️ type-aware is deprecated - TypeAware is now always enabled.'
)
console.warn(
' Just use the underlying storage type (e.g., "filesystem", "s3", etc.)'
@ -866,7 +863,7 @@ export async function createStorage(
}
/**
* Export storage adapters (v5.4.0: TypeAware is now built-in, no separate export)
* Export storage adapters (TypeAware is now built-in, no separate export)
*/
export {
MemoryStorage,