fix(storage): clean directory architecture - FileSystemStorage uses entities/nouns/hnsw and entities/verbs/hnsw

- FileSystemStorage now uses clean hardcoded paths
- Noun vectors: entities/nouns/hnsw (was: nouns/)
- Verb vectors: entities/verbs/hnsw (was: verbs/)
- Noun metadata: entities/nouns/metadata
- Verb metadata: entities/verbs/metadata
- Removed dual read/write backward compatibility code
- Removed mergeStatistics method (no longer needed)
- Added deprecation stubs for other adapters (to be migrated in v4.7.3)

This fixes VFS bug where verb vector files were written to wrong directory.
Workshop team: Delete brainy-data folder and reimport with v4.7.2.

src/storage/adapters/fileSystemStorage.ts

@@ -15,16 +15,9 @@ import {
 } from '../../coreTypes.js'
 import {
   BaseStorage,
-  NOUNS_DIR,
-  VERBS_DIR,
-  METADATA_DIR,
-  NOUN_METADATA_DIR,
-  VERB_METADATA_DIR,
-  INDEX_DIR,
   SYSTEM_DIR,
   STATISTICS_KEY
 } from '../baseStorage.js'
-import { StorageCompatibilityLayer, StoragePaths } from '../backwardCompatibility.js'
 
 // Type aliases for better readability
 type HNSWNode = HNSWNoun
@@ -84,9 +77,8 @@ export class FileSystemStorage extends BaseStorage {
   private nounMetadataDir!: string
   private verbMetadataDir!: string
   private indexDir!: string  // Legacy - for backward compatibility
-  private systemDir!: string  // New location for system data
+  private systemDir!: string
   private lockDir!: string
-  private useDualWrite: boolean = true  // Write to both locations during migration
   private activeLocks: Set<string> = new Set()
   private lockTimers: Map<string, NodeJS.Timeout> = new Map()  // Track timers for cleanup
   private allTimers: Set<NodeJS.Timeout> = new Set()  // Track all timers for cleanup
@@ -149,13 +141,14 @@ export class FileSystemStorage extends BaseStorage {
 
     try {
       // Initialize directory paths now that path module is loaded
-      this.nounsDir = path.join(this.rootDir, NOUNS_DIR)
+      // Clean directory structure (v4.7.2+)
-      this.verbsDir = path.join(this.rootDir, VERBS_DIR)
+      this.nounsDir = path.join(this.rootDir, 'entities/nouns/hnsw')
-      this.metadataDir = path.join(this.rootDir, METADATA_DIR)
+      this.verbsDir = path.join(this.rootDir, 'entities/verbs/hnsw')
-      this.nounMetadataDir = path.join(this.rootDir, NOUN_METADATA_DIR)
+      this.metadataDir = path.join(this.rootDir, 'entities/nouns/metadata')  // Legacy reference
-      this.verbMetadataDir = path.join(this.rootDir, VERB_METADATA_DIR)
+      this.nounMetadataDir = path.join(this.rootDir, 'entities/nouns/metadata')
-      this.indexDir = path.join(this.rootDir, INDEX_DIR)  // Legacy
+      this.verbMetadataDir = path.join(this.rootDir, 'entities/verbs/metadata')
-      this.systemDir = path.join(this.rootDir, SYSTEM_DIR)  // New
+      this.indexDir = path.join(this.rootDir, 'indexes')
+      this.systemDir = path.join(this.rootDir, SYSTEM_DIR)
       this.lockDir = path.join(this.rootDir, 'locks')
 
       // Create the root directory if it doesn't exist
@@ -1642,7 +1635,7 @@ export class FileSystemStorage extends BaseStorage {
 
     try {
       // Get existing statistics to merge with new data
-      const existingStats = await this.getStatisticsWithBackwardCompat()
+      const existingStats = await this.getStatisticsData()
 
       if (existingStats) {
         // Merge statistics data
@@ -1686,140 +1679,25 @@ export class FileSystemStorage extends BaseStorage {
    * Get statistics data from storage
    */
   protected async getStatisticsData(): Promise<StatisticsData | null> {
-    return this.getStatisticsWithBackwardCompat()
-  }
-
-  /**
-   * Save statistics with backward compatibility (dual write)
-   */
-  private async saveStatisticsWithBackwardCompat(statistics: StatisticsData): Promise<void> {
-    // Always write to new location
-    const newPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
-    await this.ensureDirectoryExists(this.systemDir)
-    await fs.promises.writeFile(newPath, JSON.stringify(statistics, null, 2))
-    
-    // During migration period, also write to old location if it exists
-    if (this.useDualWrite && await this.directoryExists(this.indexDir)) {
-      const oldPath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
-      try {
-        await fs.promises.writeFile(oldPath, JSON.stringify(statistics, null, 2))
-      } catch (error) {
-        // Log but don't fail if old location write fails
-        StorageCompatibilityLayer.logMigrationEvent(
-          'Failed to write to legacy location',
-          { path: oldPath, error }
-        )
-      }
-    }
-  }
-
-  /**
-   * Get statistics with backward compatibility (dual read)
-   */
-  private async getStatisticsWithBackwardCompat(): Promise<StatisticsData | null> {
-    let newStats: StatisticsData | null = null
-    let oldStats: StatisticsData | null = null
-    
-    // Try to read from new location first
     try {
-      const newPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
+      const statsPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
-      const data = await fs.promises.readFile(newPath, 'utf-8')
+      const data = await fs.promises.readFile(statsPath, 'utf-8')
-      newStats = JSON.parse(data)
+      return JSON.parse(data)
     } catch (error: any) {
       if (error.code !== 'ENOENT') {
-        console.error('Error reading statistics from new location:', error)
+        console.error('Error reading statistics:', error)
       }
+      return null
     }
-    
-    // Try to read from old location as fallback
-    if (!newStats && await this.directoryExists(this.indexDir)) {
-      try {
-        const oldPath = path.join(this.indexDir, `${STATISTICS_KEY}.json`)
-        const data = await fs.promises.readFile(oldPath, 'utf-8')
-        oldStats = JSON.parse(data)
-        
-        // If we found data in old location but not new, migrate it
-        if (oldStats && !newStats) {
-          StorageCompatibilityLayer.logMigrationEvent(
-            'Migrating statistics from legacy location'
-          )
-          await this.saveStatisticsWithBackwardCompat(oldStats)
-        }
-      } catch (error: any) {
-        if (error.code !== 'ENOENT') {
-          console.error('Error reading statistics from old location:', error)
-        }
-      }
-    }
-    
-    // Merge statistics from both locations
-    return this.mergeStatistics(newStats, oldStats)
   }
 
   /**
-   * Merge statistics from multiple sources
+   * Save statistics to storage
    */
-  private mergeStatistics(
+  private async saveStatisticsWithBackwardCompat(statistics: StatisticsData): Promise<void> {
-    storageStats: StatisticsData | null,
+    const statsPath = path.join(this.systemDir, `${STATISTICS_KEY}.json`)
-    localStats: StatisticsData | null
+    await this.ensureDirectoryExists(this.systemDir)
-  ): StatisticsData {
+    await fs.promises.writeFile(statsPath, JSON.stringify(statistics, null, 2))
-    // Handle null cases
-    if (!storageStats && !localStats) {
-      // CRITICAL FIX (v3.37.4): Statistics files don't exist yet (first init)
-      // Return minimal stats with counts instead of zeros
-      // This prevents HNSW from seeing entityCount=0 during index rebuild
-      return {
-        nounCount: {},
-        verbCount: {},
-        metadataCount: {},
-        hnswIndexSize: 0,
-        totalNodes: this.totalNounCount,
-        totalEdges: this.totalVerbCount,
-        totalMetadata: 0,
-        lastUpdated: new Date().toISOString()
-      }
-    }
-    if (!storageStats) return localStats!
-    if (!localStats) return storageStats
-
-    // Merge noun counts by taking the maximum of each type
-    const mergedNounCount: Record<string, number> = {
-      ...storageStats.nounCount
-    }
-    for (const [type, count] of Object.entries(localStats.nounCount)) {
-      mergedNounCount[type] = Math.max(mergedNounCount[type] || 0, count)
-    }
-
-    // Merge verb counts by taking the maximum of each type
-    const mergedVerbCount: Record<string, number> = {
-      ...storageStats.verbCount
-    }
-    for (const [type, count] of Object.entries(localStats.verbCount)) {
-      mergedVerbCount[type] = Math.max(mergedVerbCount[type] || 0, count)
-    }
-
-    // Merge metadata counts by taking the maximum of each type
-    const mergedMetadataCount: Record<string, number> = {
-      ...storageStats.metadataCount
-    }
-    for (const [type, count] of Object.entries(localStats.metadataCount)) {
-      mergedMetadataCount[type] = Math.max(
-        mergedMetadataCount[type] || 0,
-        count
-      )
-    }
-
-    return {
-      nounCount: mergedNounCount,
-      verbCount: mergedVerbCount,
-      metadataCount: mergedMetadataCount,
-      hnswIndexSize: Math.max(storageStats.hnswIndexSize || 0, localStats.hnswIndexSize || 0),
-      totalNodes: Math.max(storageStats.totalNodes || 0, localStats.totalNodes || 0),
-      totalEdges: Math.max(storageStats.totalEdges || 0, localStats.totalEdges || 0),
-      totalMetadata: Math.max(storageStats.totalMetadata || 0, localStats.totalMetadata || 0),
-      operations: storageStats.operations || localStats.operations,
-      lastUpdated: new Date().toISOString()
-    }
   }
 
   // =============================================

src/storage/backwardCompatibility.ts

@@ -1,17 +1,15 @@
 /**
- * Storage backward compatibility layer for legacy data migrations
+ * DEPRECATED (v4.7.2): Backward compatibility stubs
+ * TODO: Remove in v4.7.3 after migrating s3CompatibleStorage
  */
 
 export class StorageCompatibilityLayer {
   static logMigrationEvent(event: string, details?: any): void {
-    // Simplified logging for migration events
+    // No-op
-    if (process.env.DEBUG_MIGRATION) {
-      console.log(`[Migration] ${event}`, details)
-    }
   }
 
   static async migrateIfNeeded(storagePath: string): Promise<void> {
-    // No-op for now - can be extended later if needed
+    // No-op
   }
 }
 
@@ -24,14 +22,13 @@ export interface StoragePaths {
   statistics: string
 }
 
-// Helper to get default paths
 export function getDefaultStoragePaths(basePath: string): StoragePaths {
   return {
-    nouns: `${basePath}/nouns`,
+    nouns: `${basePath}/entities/nouns/hnsw`,
-    verbs: `${basePath}/verbs`, 
+    verbs: `${basePath}/entities/verbs/hnsw`,
-    metadata: `${basePath}/metadata`,
+    metadata: `${basePath}/entities/nouns/metadata`,
-    index: `${basePath}/index`,
+    index: `${basePath}/indexes`,
-    system: `${basePath}/system`,
+    system: `${basePath}/_system`,
-    statistics: `${basePath}/statistics.json`
+    statistics: `${basePath}/_system/statistics.json`
   }
 }

src/storage/baseStorage.ts

@@ -32,50 +32,26 @@ interface StorageKeyInfo {
   fullPath: string
 }
 
-// Common directory/prefix names
+// Clean directory structure (v4.7.2+)
-// Option A: Entity-Based Directory Structure
+// All storage adapters use this consistent structure
-export const ENTITIES_DIR = 'entities'
-export const NOUNS_VECTOR_DIR = 'entities/nouns/vectors'
 export const NOUNS_METADATA_DIR = 'entities/nouns/metadata'
-export const VERBS_VECTOR_DIR = 'entities/verbs/vectors'
 export const VERBS_METADATA_DIR = 'entities/verbs/metadata'
-export const INDEXES_DIR = 'indexes'
+export const SYSTEM_DIR = '_system'
-export const METADATA_INDEX_DIR = 'indexes/metadata'
-
-// Legacy paths - kept for backward compatibility during migration
-export const NOUNS_DIR = 'nouns'  // Legacy: now maps to entities/nouns/vectors
-export const VERBS_DIR = 'verbs'  // Legacy: now maps to entities/verbs/vectors
-export const METADATA_DIR = 'metadata'  // Legacy: now maps to entities/nouns/metadata
-export const NOUN_METADATA_DIR = 'noun-metadata'  // Legacy: now maps to entities/nouns/metadata
-export const VERB_METADATA_DIR = 'verb-metadata'  // Legacy: now maps to entities/verbs/metadata
-export const INDEX_DIR = 'index'  // Legacy - kept for backward compatibility
-export const SYSTEM_DIR = '_system'  // System config & metadata indexes
 export const STATISTICS_KEY = 'statistics'
 
-// Migration version to track compatibility
+// DEPRECATED (v4.7.2): Temporary stubs for adapters not yet migrated
-export const STORAGE_SCHEMA_VERSION = 3  // v3: Entity-Based Directory Structure (Option A)
+// TODO: Remove in v4.7.3 after migrating remaining adapters
-
+export const NOUNS_DIR = 'entities/nouns/hnsw'
-// Configuration flag to enable new directory structure
+export const VERBS_DIR = 'entities/verbs/hnsw'
-export const USE_ENTITY_BASED_STRUCTURE = true  // Set to true to use Option A structure
+export const METADATA_DIR = 'entities/nouns/metadata'
-
+export const NOUN_METADATA_DIR = 'entities/nouns/metadata'
-/**
+export const VERB_METADATA_DIR = 'entities/verbs/metadata'
- * Get the appropriate directory path based on configuration
+export const INDEX_DIR = 'indexes'
- */
 export function getDirectoryPath(entityType: 'noun' | 'verb', dataType: 'vector' | 'metadata'): string {
-  if (USE_ENTITY_BASED_STRUCTURE) {
+  if (entityType === 'noun') {
-    // Option A: Entity-Based Structure
+    return dataType === 'vector' ? NOUNS_DIR : NOUNS_METADATA_DIR
-    if (entityType === 'noun') {
-      return dataType === 'vector' ? NOUNS_VECTOR_DIR : NOUNS_METADATA_DIR
-    } else {
-      return dataType === 'vector' ? VERBS_VECTOR_DIR : VERBS_METADATA_DIR  
-    }
   } else {
-    // Legacy structure
+    return dataType === 'vector' ? VERBS_DIR : VERBS_METADATA_DIR
-    if (entityType === 'noun') {
-      return dataType === 'vector' ? NOUNS_DIR : METADATA_DIR
-    } else {
-      return dataType === 'vector' ? VERBS_DIR : VERB_METADATA_DIR
-    }
   }
 }