feat: add distributed architecture with sharding and coordination

- Wire up distributed components (Coordinator, ShardManager, CacheSync)
- Implement automatic sharding for S3 storage (256 shards)
- Add read/write separation for operational modes
- Zero-config automatic detection for distributed mode
- Add mutex implementation for thread safety
- Fix metadata filtering in find operations
- Fix neural API vector similarity calculations
- Improve batch operations performance
- Add Bluesky distributed setup example

BREAKING CHANGE: None - backward compatible

src/utils/metadataIndex.ts

@@ -112,16 +112,41 @@ export class MetadataIndexManager {
       indexedFields: config.indexedFields ?? [],
       excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt', 'embedding', 'vector', 'embeddings', 'vectors']
     }
-    
+
     // Initialize metadata cache with similar config to search cache
     this.metadataCache = new MetadataIndexCache({
       maxAge: 5 * 60 * 1000, // 5 minutes
       maxSize: 500,          // 500 entries (field indexes + value chunks)
       enabled: true
     })
-    
+
     // Get global unified cache for coordinated memory management
     this.unifiedCache = getGlobalCache()
+
+    // Lazy load counts from storage statistics on first access
+    this.lazyLoadCounts()
+  }
+
+  /**
+   * Lazy load entity counts from storage statistics (O(1) operation)
+   * This avoids rebuilding the entire index on startup
+   */
+  private async lazyLoadCounts(): Promise<void> {
+    try {
+      // Get statistics from storage (should be O(1) with our FileSystemStorage improvements)
+      const stats = await this.storage.getStatistics()
+      if (stats && stats.nounCount) {
+        // Populate entity counts from storage statistics
+        for (const [type, count] of Object.entries(stats.nounCount)) {
+          if (typeof count === 'number' && count > 0) {
+            this.totalEntitiesByType.set(type, count)
+          }
+        }
+      }
+    } catch (error) {
+      // Silently fail - counts will be populated as entities are added
+      // This maintains zero-configuration principle
+    }
   }
 
   /**

src/utils/mutex.ts

@@ -0,0 +1,272 @@
+/**
+ * Universal Mutex Implementation for Thread-Safe Operations
+ * Provides consistent locking across all storage adapters
+ * Critical for preventing race conditions in count operations
+ */
+
+export interface MutexInterface {
+  acquire(key: string, timeout?: number): Promise<() => void>
+  runExclusive<T>(key: string, fn: () => Promise<T>, timeout?: number): Promise<T>
+  isLocked(key: string): boolean
+}
+
+/**
+ * In-memory mutex for single-process scenarios
+ * Used by MemoryStorage and as fallback for other adapters
+ */
+export class InMemoryMutex implements MutexInterface {
+  private locks: Map<string, {
+    queue: Array<() => void>
+    locked: boolean
+  }> = new Map()
+
+  async acquire(key: string, timeout: number = 30000): Promise<() => void> {
+    if (!this.locks.has(key)) {
+      this.locks.set(key, { queue: [], locked: false })
+    }
+
+    const lock = this.locks.get(key)!
+
+    if (!lock.locked) {
+      lock.locked = true
+      return () => this.release(key)
+    }
+
+    // Wait in queue
+    return new Promise<() => void>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        const index = lock.queue.indexOf(resolver)
+        if (index !== -1) {
+          lock.queue.splice(index, 1)
+        }
+        reject(new Error(`Mutex timeout for key: ${key}`))
+      }, timeout)
+
+      const resolver = () => {
+        clearTimeout(timer)
+        lock.locked = true
+        resolve(() => this.release(key))
+      }
+
+      lock.queue.push(resolver)
+    })
+  }
+
+  private release(key: string): void {
+    const lock = this.locks.get(key)
+    if (!lock) return
+
+    if (lock.queue.length > 0) {
+      const next = lock.queue.shift()!
+      next()
+    } else {
+      lock.locked = false
+      // Clean up if no waiters
+      if (lock.queue.length === 0) {
+        this.locks.delete(key)
+      }
+    }
+  }
+
+  async runExclusive<T>(
+    key: string,
+    fn: () => Promise<T>,
+    timeout?: number
+  ): Promise<T> {
+    const release = await this.acquire(key, timeout)
+    try {
+      return await fn()
+    } finally {
+      release()
+    }
+  }
+
+  isLocked(key: string): boolean {
+    return this.locks.get(key)?.locked || false
+  }
+}
+
+/**
+ * File-based mutex for multi-process scenarios (Node.js)
+ * Uses atomic file operations to prevent TOCTOU races
+ */
+export class FileMutex implements MutexInterface {
+  private fs: any
+  private path: any
+  private lockDir: string
+  private processLocks: Map<string, () => void> = new Map()
+  private lockTimers: Map<string, NodeJS.Timeout> = new Map()
+
+  constructor(lockDir: string) {
+    this.lockDir = lockDir
+    // Lazy load Node.js modules
+    if (typeof window === 'undefined') {
+      this.fs = require('fs')
+      this.path = require('path')
+    }
+  }
+
+  async acquire(key: string, timeout: number = 30000): Promise<() => void> {
+    if (!this.fs || !this.path) {
+      throw new Error('FileMutex is only available in Node.js environments')
+    }
+
+    const lockFile = this.path.join(this.lockDir, `${key}.lock`)
+    const lockId = `${Date.now()}_${Math.random()}_${process.pid}`
+    const startTime = Date.now()
+
+    // Ensure lock directory exists
+    await this.fs.promises.mkdir(this.lockDir, { recursive: true })
+
+    while (Date.now() - startTime < timeout) {
+      try {
+        // Atomic lock creation using 'wx' flag
+        await this.fs.promises.writeFile(
+          lockFile,
+          JSON.stringify({
+            lockId,
+            pid: process.pid,
+            timestamp: Date.now(),
+            expiresAt: Date.now() + timeout
+          }),
+          { flag: 'wx' } // Write exclusive - fails if exists
+        )
+
+        // Successfully acquired lock
+        const release = () => this.release(key, lockFile, lockId)
+        this.processLocks.set(key, release)
+
+        // Auto-release on timeout
+        const timer = setTimeout(() => {
+          release()
+        }, timeout)
+        this.lockTimers.set(key, timer)
+
+        return release
+      } catch (error: any) {
+        if (error.code === 'EEXIST') {
+          // Lock exists - check if expired
+          try {
+            const data = await this.fs.promises.readFile(lockFile, 'utf-8')
+            const lock = JSON.parse(data)
+
+            if (lock.expiresAt < Date.now()) {
+              // Expired - try to remove
+              try {
+                await this.fs.promises.unlink(lockFile)
+                continue // Retry acquisition
+              } catch (unlinkError: any) {
+                if (unlinkError.code !== 'ENOENT') {
+                  // Someone else removed it, continue
+                  continue
+                }
+              }
+            }
+          } catch {
+            // Can't read lock file, assume it's valid
+          }
+
+          // Wait before retry
+          await new Promise(resolve => setTimeout(resolve, 50))
+        } else {
+          throw error
+        }
+      }
+    }
+
+    throw new Error(`Failed to acquire mutex for key: ${key} after ${timeout}ms`)
+  }
+
+  private async release(key: string, lockFile: string, lockId: string): Promise<void> {
+    // Clear timer
+    const timer = this.lockTimers.get(key)
+    if (timer) {
+      clearTimeout(timer)
+      this.lockTimers.delete(key)
+    }
+
+    // Remove from process locks
+    this.processLocks.delete(key)
+
+    try {
+      // Verify we own the lock before releasing
+      const data = await this.fs.promises.readFile(lockFile, 'utf-8')
+      const lock = JSON.parse(data)
+
+      if (lock.lockId === lockId) {
+        await this.fs.promises.unlink(lockFile)
+      }
+    } catch {
+      // Lock already released or doesn't exist
+    }
+  }
+
+  async runExclusive<T>(
+    key: string,
+    fn: () => Promise<T>,
+    timeout?: number
+  ): Promise<T> {
+    const release = await this.acquire(key, timeout)
+    try {
+      return await fn()
+    } finally {
+      release()
+    }
+  }
+
+  isLocked(key: string): boolean {
+    return this.processLocks.has(key)
+  }
+
+  /**
+   * Clean up all locks held by this process
+   */
+  async cleanup(): Promise<void> {
+    // Clear all timers
+    for (const timer of this.lockTimers.values()) {
+      clearTimeout(timer)
+    }
+    this.lockTimers.clear()
+
+    // Release all locks
+    const releases = Array.from(this.processLocks.values())
+    await Promise.all(releases.map(release => release()))
+    this.processLocks.clear()
+  }
+}
+
+/**
+ * Factory to create appropriate mutex for the environment
+ */
+export function createMutex(options?: {
+  type?: 'memory' | 'file'
+  lockDir?: string
+}): MutexInterface {
+  const type = options?.type || (typeof window === 'undefined' ? 'file' : 'memory')
+
+  if (type === 'file' && typeof window === 'undefined') {
+    const lockDir = options?.lockDir || '.brainy/locks'
+    return new FileMutex(lockDir)
+  }
+
+  return new InMemoryMutex()
+}
+
+// Global mutex instance for count operations
+let globalMutex: MutexInterface | null = null
+
+export function getGlobalMutex(): MutexInterface {
+  if (!globalMutex) {
+    globalMutex = createMutex()
+  }
+  return globalMutex
+}
+
+/**
+ * Cleanup function for graceful shutdown
+ */
+export async function cleanupMutexes(): Promise<void> {
+  if (globalMutex && 'cleanup' in globalMutex) {
+    await (globalMutex as any).cleanup()
+  }
+}

src/utils/paramValidation.ts

@@ -165,12 +165,29 @@ export function validateFindParams(params: FindParams): void {
 export function validateAddParams(params: AddParams): void {
   // Universal truth: must have data or vector
   if (!params.data && !params.vector) {
-    throw new Error('must provide either data or vector')
+    throw new Error(
+      `Invalid add() parameters: Missing required field 'data'\n` +
+      `\nReceived: ${JSON.stringify({
+        type: params.type,
+        hasMetadata: !!params.metadata,
+        hasId: !!params.id
+      }, null, 2)}\n` +
+      `\nExpected one of:\n` +
+      `  { data: 'text to store', type?: 'note', metadata?: {...} }\n` +
+      `  { vector: [0.1, 0.2, ...], type?: 'embedding', metadata?: {...} }\n` +
+      `\nExamples:\n` +
+      `  await brain.add({ data: 'Machine learning is AI', type: 'concept' })\n` +
+      `  await brain.add({ data: { title: 'Doc', content: '...' }, type: 'document' })`
+    )
   }
   
   // Validate noun type
   if (!Object.values(NounType).includes(params.type)) {
-    throw new Error(`invalid NounType: ${params.type}`)
+    throw new Error(
+      `Invalid NounType: '${params.type}'\n` +
+      `\nValid types: ${Object.values(NounType).join(', ')}\n` +
+      `\nExample: await brain.add({ data: 'text', type: NounType.Note })`
+    )
   }
   
   // Validate vector dimensions if provided
@@ -227,8 +244,10 @@ export function validateRelateParams(params: RelateParams): void {
     throw new Error('cannot create self-referential relationship')
   }
   
-  // Validate verb type
-  if (!Object.values(VerbType).includes(params.type)) {
+  // Validate verb type - default to RelatedTo if not specified
+  if (params.type === undefined) {
+    params.type = VerbType.RelatedTo
+  } else if (!Object.values(VerbType).includes(params.type)) {
     throw new Error(`invalid VerbType: ${params.type}`)
   }