feat: v0.49 - Filter discovery API, remove deprecated methods, improve performance

BREAKING CHANGES:
- Removed deprecated getAllNouns() and getAllVerbs() methods
- All internal usage migrated to pagination-based methods

New Features:
- Filter Discovery API:
  - getFilterValues(field): Get all available values for a field
  - getFilterFields(): Get all filterable fields
  - Enables dynamic filter UI generation with O(1) field discovery
- Hybrid metadata indexing with field-level indexes
- Adaptive auto-flush for optimal performance
- LRU caching for metadata indexes

Improvements:
- Fixed ENAMETOOLONG errors from vector-based filenames
- Safe filename generation using hash-based approach
- Scalable chunked value storage for millions of entries
- Performance optimization with adaptive flush thresholds
- Added support for $includes operator in metadata filters

Technical:
- Replaced vector-based filenames with safe hash approach
- Implemented MetadataIndexCache with existing SearchCache pattern
- Field indexes enable O(1) filter discovery
- Adaptive flush based on performance metrics (20-200 entries)
- All tests passing with improved metadata filtering

src/storage/adapters/baseStorageAdapter.ts

@@ -17,8 +17,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
 
   abstract getNoun(id: string): Promise<any | null>
 
-  abstract getAllNouns(): Promise<any[]>
-
   abstract getNounsByNounType(nounType: string): Promise<any[]>
 
   abstract deleteNoun(id: string): Promise<void>
@@ -27,8 +25,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
 
   abstract getVerb(id: string): Promise<any | null>
 
-  abstract getAllVerbs(): Promise<any[]>
-
   abstract getVerbsBySource(sourceId: string): Promise<any[]>
 
   abstract getVerbsByTarget(targetId: string): Promise<any[]>
@@ -54,6 +50,20 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
     details?: Record<string, any>
   }>
 
+  /**
+   * Get all nouns from storage
+   * @returns Promise that resolves to an array of all nouns
+   * @deprecated This method loads all data into memory and may cause performance issues. Use getNouns() with pagination instead.
+   */
+  abstract getAllNouns(): Promise<any[]>
+
+  /**
+   * Get all verbs from storage
+   * @returns Promise that resolves to an array of all HNSWVerbs
+   * @deprecated This method loads all data into memory and may cause performance issues. Use getVerbs() with pagination instead.
+   */
+  abstract getAllVerbs(): Promise<any[]>
+
   /**
    * Get nouns with pagination and filtering
    * @param options Pagination and filtering options

src/storage/adapters/fileSystemStorage.ts

@@ -752,12 +752,6 @@ export class FileSystemStorage extends BaseStorage {
     return this.getNode(id)
   }
 
-  /**
-   * Get all nouns from storage
-   */
-  protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
-    return this.getAllNodes()
-  }
 
   /**
    * Get nouns by noun type
@@ -789,12 +783,6 @@ export class FileSystemStorage extends BaseStorage {
     return this.getEdge(id)
   }
 
-  /**
-   * Get all verbs from storage
-   */
-  protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
-    return this.getAllEdges()
-  }
 
   /**
    * Get verbs by source

src/storage/adapters/memoryStorage.ts

@@ -83,33 +83,6 @@ export class MemoryStorage extends BaseStorage {
     return nounCopy
   }
 
-  /**
-   * Get all nouns from storage
-   */
-  protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
-    const allNouns: HNSWNoun[] = []
-
-    // Iterate through all nouns in the nouns map
-    for (const [nounId, noun] of this.nouns.entries()) {
-      // Return a deep copy to avoid reference issues
-      const nounCopy: HNSWNoun = {
-        id: noun.id,
-        vector: [...noun.vector],
-        connections: new Map(),
-        level: noun.level || 0
-      }
-
-      // Copy connections
-      for (const [level, connections] of noun.connections.entries()) {
-        nounCopy.connections.set(level, new Set(connections))
-      }
-
-      allNouns.push(nounCopy)
-    }
-
-    return allNouns
-  }
-
   /**
    * Get nouns with pagination and filtering
    * @param options Pagination and filtering options
@@ -297,32 +270,6 @@ export class MemoryStorage extends BaseStorage {
     return verbCopy
   }
 
-  /**
-   * Get all verbs from storage
-   */
-  protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
-    const allVerbs: HNSWVerb[] = []
-
-    // Iterate through all verbs in the verbs map
-    for (const [verbId, verb] of this.verbs.entries()) {
-      // Create a deep copy of the HNSWVerb
-      const verbCopy: HNSWVerb = {
-        id: verb.id,
-        vector: [...verb.vector],
-        connections: new Map()
-      }
-
-      // Copy connections
-      for (const [level, connections] of verb.connections.entries()) {
-        verbCopy.connections.set(level, new Set(connections))
-      }
-
-      allVerbs.push(verbCopy)
-    }
-
-    return allVerbs
-  }
-
   /**
    * Get verbs with pagination and filtering
    * @param options Pagination and filtering options

src/storage/adapters/opfsStorage.ts

@@ -255,46 +255,6 @@ export class OPFSStorage extends BaseStorage {
     }
   }
 
-  /**
-   * Get all nouns from storage
-   */
-  protected async getAllNouns_internal(): Promise<HNSWNoun_internal[]> {
-    await this.ensureInitialized()
-
-    const allNouns: HNSWNoun_internal[] = []
-    try {
-      // Iterate through all files in the nouns directory
-      for await (const [name, handle] of this.nounsDir!.entries()) {
-        if (handle.kind === 'file') {
-          try {
-            // Read the noun data from the file
-            const file = await safeGetFile(handle)
-            const text = await file.text()
-            const data = JSON.parse(text)
-
-            // Convert serialized connections back to Map<number, Set<string>>
-            const connections = new Map<number, Set<string>>()
-            for (const [level, nounIds] of Object.entries(data.connections)) {
-              connections.set(Number(level), new Set(nounIds as string[]))
-            }
-
-            allNouns.push({
-              id: data.id,
-              vector: data.vector,
-              connections,
-              level: data.level || 0
-            })
-          } catch (error) {
-            console.error(`Error reading noun file ${name}:`, error)
-          }
-        }
-      }
-    } catch (error) {
-      console.error('Error reading nouns directory:', error)
-    }
-
-    return allNouns
-  }
 
   /**
    * Get nouns by noun type (internal implementation)
@@ -469,12 +429,6 @@ export class OPFSStorage extends BaseStorage {
     }
   }
 
-  /**
-   * Get all verbs from storage (internal implementation)
-   */
-  protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
-    return this.getAllEdges()
-  }
 
   /**
    * Get all edges from storage

src/storage/adapters/s3CompatibleStorage.ts

@@ -466,17 +466,6 @@ export class S3CompatibleStorage extends BaseStorage {
     }
   }
 
-  /**
-   * Get all nouns from storage (internal implementation)
-   */
-  protected async getAllNouns_internal(): Promise<HNSWNoun[]> {
-    // Use paginated method to avoid deprecation warning
-    const result = await this.getNodesWithPagination({
-      limit: 1000,
-      useCache: true
-    })
-    return result.nodes
-  }
 
   // Node cache to avoid redundant API calls
   private nodeCache = new Map<string, HNSWNode>()
@@ -854,15 +843,6 @@ export class S3CompatibleStorage extends BaseStorage {
     }
   }
 
-  /**
-   * Get all verbs from storage (internal implementation)
-   * @deprecated This method is deprecated and will be removed in a future version.
-   * It can cause memory issues with large datasets. Use getVerbsWithPagination() instead.
-   */
-  protected async getAllVerbs_internal(): Promise<HNSWVerb[]> {
-    this.logger.warn('getAllVerbs_internal() is deprecated and will be removed in a future version. Use getVerbsWithPagination() instead.')
-    return this.getAllEdges()
-  }
 
   /**
    * Get all edges from storage