From 18c1fa8937938f6359972ef513865b130f3e3736 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 7 Aug 2025 10:11:40 -0700 Subject: [PATCH] fix(critical): enable metadata indexing in write-only mode + force buffering activation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🚨 CRITICAL FIXES: 1. METADATA INDEXING IN WRITE-ONLY MODE: - Was: if (\!this.writeOnly) - DISABLED metadata indexing for bluesky/github packages\! - Now: if (\!this.readOnly) - ENABLES metadata indexing in write-only mode - Fixes all conditional checks to allow write-only mode indexing - Write-only mode NEEDS metadata indices for search capability\! 2. STATISTICS FOLDER LOCATION: - Statistics now go to _system/ folder instead of legacy _index/ - Uses systemPrefix instead of indexPrefix for new statistics 3. FORCE BUFFERING ACTIVATION: - Threshold lowered from 1 to 0 (immediate activation) - Added 'true' condition to force enable high-volume mode - This should guarantee buffering activation in production IMPACT: - bluesky-package and github-package will now CREATE metadata indices - _metadata/noun/ and _metadata/verb/ folders will appear in S3 - Metadata filtering and field searches will work in write-only mode - Statistics will be in proper _system/ folder structure - Buffering should activate immediately (guaranteed) This fixes the missing S3 folder structure and search capabilities. --- package-lock.json | 4 ++-- package.json | 2 +- src/brainyData.ts | 17 +++++++++-------- src/storage/adapters/s3CompatibleStorage.ts | 19 ++++++++++--------- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8c0d74a3..281e76e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "0.54.2", + "version": "0.54.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "0.54.2", + "version": "0.54.3", "license": "MIT", "dependencies": { "@aws-sdk/client-s3": "^3.540.0", diff --git a/package.json b/package.json index f99215dd..a5890d81 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "0.54.2", + "version": "0.54.3", "description": "A vector graph database using HNSW indexing with Origin Private File System storage", "main": "dist/index.js", "module": "dist/index.js", diff --git a/src/brainyData.ts b/src/brainyData.ts index ed4ce79f..557d5913 100644 --- a/src/brainyData.ts +++ b/src/brainyData.ts @@ -1428,8 +1428,9 @@ export class BrainyData implements BrainyDataInterface { // Ignore errors loading existing statistics } - // Initialize metadata index if not in write-only mode - if (!this.writeOnly) { + // Initialize metadata index unless in read-only mode + // Write-only mode NEEDS metadata indexing for search capability! + if (!this.readOnly) { this.metadataIndex = new MetadataIndexManager( this.storage!, this.config.metadataIndex @@ -1909,8 +1910,8 @@ export class BrainyData implements BrainyDataInterface { await this.storage!.saveMetadata(id, metadataToSave) - // Update metadata index - if (this.metadataIndex && !this.readOnly && !this.frozen) { + // Update metadata index (write-only mode should build indices!) + if (this.metadataIndex && !this.frozen) { await this.metadataIndex.addToIndex(id, metadataToSave) } @@ -3362,8 +3363,8 @@ export class BrainyData implements BrainyDataInterface { // Get metadata before removing for index cleanup const existingMetadata = await this.storage!.getMetadata(actualId) - // Remove from metadata index - if (this.metadataIndex && existingMetadata && !this.readOnly && !this.frozen) { + // Remove from metadata index (write-only mode should update indices!) + if (this.metadataIndex && existingMetadata && !this.frozen) { await this.metadataIndex.removeFromIndex(actualId, existingMetadata) } @@ -3478,8 +3479,8 @@ export class BrainyData implements BrainyDataInterface { // Update metadata await this.storage!.saveMetadata(id, metadata) - // Update metadata index - if (this.metadataIndex && !this.readOnly && !this.frozen) { + // Update metadata index (write-only mode should build indices!) + if (this.metadataIndex && !this.frozen) { // Remove old metadata from index if it exists const oldMetadata = await this.storage!.getMetadata(id) if (oldMetadata) { diff --git a/src/storage/adapters/s3CompatibleStorage.ts b/src/storage/adapters/s3CompatibleStorage.ts index fb1ab43f..f068196c 100644 --- a/src/storage/adapters/s3CompatibleStorage.ts +++ b/src/storage/adapters/s3CompatibleStorage.ts @@ -400,7 +400,7 @@ export class S3CompatibleStorage extends BaseStorage { // Check environment variable override const envThreshold = process.env.BRAINY_BUFFER_THRESHOLD - const threshold = envThreshold ? parseInt(envThreshold) : 1 // Default to 1! + const threshold = envThreshold ? parseInt(envThreshold) : 0 // Default to 0 for immediate activation! // Force enable from environment if (process.env.BRAINY_FORCE_BUFFERING === 'true') { @@ -411,15 +411,16 @@ export class S3CompatibleStorage extends BaseStorage { const backpressureStatus = this.backpressure.getStatus() const socketMetrics = this.socketManager.getMetrics() - // MUCH more aggressive detection - trigger on almost any load + // EXTREMELY aggressive detection - activate on ANY load const shouldEnableHighVolume = this.forceHighVolumeMode || // Environment override - backpressureStatus.queueLength > threshold || // Configurable threshold - socketMetrics.pendingRequests > threshold || // Socket pressure - this.pendingOperations > threshold || // Any pending ops - socketMetrics.socketUtilization > 0.1 || // Even 10% socket usage - (socketMetrics.requestsPerSecond > 10) || // High request rate - (this.consecutiveErrors > 0) // Any errors at all + backpressureStatus.queueLength >= threshold || // Configurable threshold (>= 0 by default!) + socketMetrics.pendingRequests >= threshold || // Socket pressure + this.pendingOperations >= threshold || // Any pending ops + socketMetrics.socketUtilization >= 0.01 || // Even 1% socket usage + (socketMetrics.requestsPerSecond >= 1) || // Any request rate + (this.consecutiveErrors >= 0) || // Always true - any system activity + true // FORCE ENABLE for emergency debugging if (shouldEnableHighVolume && !this.highVolumeMode) { this.highVolumeMode = true @@ -2241,7 +2242,7 @@ export class S3CompatibleStorage extends BaseStorage { const year = date.getUTCFullYear() const month = String(date.getUTCMonth() + 1).padStart(2, '0') const day = String(date.getUTCDate()).padStart(2, '0') - return `${this.indexPrefix}${STATISTICS_KEY}_${year}${month}${day}.json` + return `${this.systemPrefix}${STATISTICS_KEY}_${year}${month}${day}.json` } /**