feat: add external plugin loader and examples for Brainy

Introduced `pluginLoader.ts` to enable loading and configuring augmentation plugins from external packages. Added `examples/externalPlugins.js` to demonstrate usage. Enhanced storage status reporting in `opfsStorage` and `BrainyData` with detailed storage capacity and usage methods. Updated README with an external plugin usage guide.

src/brainyData.ts

@@ -6,7 +6,16 @@
 import { v4 as uuidv4 } from 'uuid'
 import { HNSWIndex } from './hnsw/hnswIndex.js'
 import { createStorage } from './storage/opfsStorage.js'
-import { DistanceFunction, Edge, EmbeddingFunction, HNSWConfig, SearchResult, StorageAdapter, Vector, VectorDocument } from './coreTypes.js'
+import {
+  DistanceFunction,
+  Edge,
+  EmbeddingFunction,
+  HNSWConfig,
+  SearchResult,
+  StorageAdapter,
+  Vector,
+  VectorDocument
+} from './coreTypes.js'
 import { cosineDistance, defaultEmbeddingFunction, euclideanDistance } from './utils/index.js'
 
 export interface BrainyDataConfig {
@@ -181,9 +190,9 @@ export class BrainyData<T = any> {
    * @returns Array of IDs for the added items
    */
   public async addBatch(
-    items: Array<{ 
-      vectorOrData: Vector | any; 
-      metadata?: T 
+    items: Array<{
+      vectorOrData: Vector | any;
+      metadata?: T
     }>,
     options: {
       forceEmbed?: boolean // Force using the embedding function even if input is a vector
@@ -550,7 +559,7 @@ export class BrainyData<T = any> {
   /**
    * Embed text or data into a vector using the same embedding function used by this instance
    * This allows clients to use the same TensorFlow Universal Sentence Encoder throughout their application
-   * 
+   *
    * @param data Text or data to embed
    * @returns A promise that resolves to the embedded vector
    */
@@ -568,7 +577,7 @@ export class BrainyData<T = any> {
   /**
    * Search for similar documents using a text query
    * This is a convenience method that embeds the query text and performs a search
-   * 
+   *
    * @param query Text query to search for
    * @param k Number of results to return
    * @returns Array of search results
@@ -596,6 +605,54 @@ export class BrainyData<T = any> {
       await this.init()
     }
   }
+
+  /**
+   * Get information about the current storage usage and capacity
+   * @returns Object containing the storage type, used space, quota, and additional details
+   */
+  public async status(): Promise<{
+    type: string;
+    used: number;
+    quota: number | null;
+    details?: Record<string, any>;
+  }> {
+    await this.ensureInitialized()
+
+    if (!this.storage) {
+      return {
+        type: 'unknown',
+        used: 0,
+        quota: null,
+        details: { error: 'Storage not initialized' }
+      }
+    }
+
+    try {
+      // Get storage status from the storage adapter
+      const storageStatus = await this.storage.getStorageStatus()
+
+      // Add index information to the details
+      const indexInfo = {
+        indexSize: this.size()
+      }
+
+      return {
+        ...storageStatus,
+        details: {
+          ...storageStatus.details,
+          index: indexInfo
+        }
+      }
+    } catch (error) {
+      console.error('Failed to get storage status:', error)
+      return {
+        type: 'unknown',
+        used: 0,
+        quota: null,
+        details: { error: String(error) }
+      }
+    }
+  }
 }
 
 // Export distance functions for convenience

src/coreTypes.ts

@@ -120,4 +120,30 @@ export interface StorageAdapter {
   getMetadata(id: string): Promise<any | null>;
 
   clear(): Promise<void>;
+
+  /**
+   * Get information about storage usage and capacity
+   * @returns Promise that resolves to an object containing storage status information
+   */
+  getStorageStatus(): Promise<{
+    /**
+     * The type of storage being used (e.g., 'filesystem', 'opfs', 'memory')
+     */
+    type: string;
+
+    /**
+     * The amount of storage being used in bytes
+     */
+    used: number;
+
+    /**
+     * The total amount of storage available in bytes, or null if unknown
+     */
+    quota: number | null;
+
+    /**
+     * Additional storage-specific information
+     */
+    details?: Record<string, any>;
+  }>;
 }

src/index.ts

@@ -66,6 +66,30 @@ export {
 }
 export type { PipelineOptions }
 
+// Export plugin loader
+import {
+  loadPlugins,
+  configureAndStartPipeline,
+  createSensePluginConfig,
+  createConduitPluginConfig
+} from './pluginLoader.js'
+import type {
+  PluginLoaderOptions,
+  PluginConfig,
+  PluginLoadResult
+} from './pluginLoader.js'
+export {
+  loadPlugins,
+  configureAndStartPipeline,
+  createSensePluginConfig,
+  createConduitPluginConfig
+}
+export type {
+  PluginLoaderOptions,
+  PluginConfig,
+  PluginLoadResult
+}
+
 // Export types
 import type {
   Vector,

src/pluginLoader.ts

@@ -0,0 +1,241 @@
+/**
+ * Plugin Loader for Brainy Augmentations
+ * 
+ * This module provides functionality for loading and configuring augmentation plugins
+ * from external npm packages. It allows consumers of the library to easily integrate
+ * externally installed augmentation plugins into the Brainy system.
+ */
+
+import { AugmentationPipeline, augmentationPipeline, ExecutionMode } from './augmentationPipeline.js';
+import { AugmentationType, IAugmentation, ISenseAugmentation, IConduitAugmentation } from './types/augmentations.js';
+
+/**
+ * Configuration options for loading plugins
+ */
+export interface PluginLoaderOptions {
+  /**
+   * Whether to use the default augmentation pipeline instance
+   * If false, a new pipeline instance will be created
+   * @default true
+   */
+  useDefaultPipeline?: boolean;
+
+  /**
+   * Whether to initialize the augmentations after loading
+   * @default true
+   */
+  initializeAfterLoading?: boolean;
+
+  /**
+   * Order of augmentation types to load
+   * By default, sense augmentations are loaded first
+   * @default [AugmentationType.SENSE, AugmentationType.CONDUIT, ...]
+   */
+  augmentationOrder?: AugmentationType[];
+}
+
+/**
+ * Default options for the plugin loader
+ */
+const DEFAULT_OPTIONS: PluginLoaderOptions = {
+  useDefaultPipeline: true,
+  initializeAfterLoading: true,
+  augmentationOrder: [
+    AugmentationType.SENSE,
+    AugmentationType.CONDUIT,
+    AugmentationType.COGNITION,
+    AugmentationType.MEMORY,
+    AugmentationType.PERCEPTION,
+    AugmentationType.DIALOG,
+    AugmentationType.ACTIVATION,
+    AugmentationType.WEBSOCKET
+  ]
+};
+
+/**
+ * Plugin configuration for a specific augmentation
+ */
+export interface PluginConfig {
+  /**
+   * The plugin module name or path
+   * This can be an npm package name or a relative path
+   */
+  plugin: string;
+
+  /**
+   * Optional configuration to pass to the plugin
+   */
+  config?: Record<string, any>;
+
+  /**
+   * Optional type of the augmentation
+   * If not provided, it will be determined automatically
+   */
+  type?: AugmentationType;
+}
+
+/**
+ * Result of loading plugins
+ */
+export interface PluginLoadResult {
+  /**
+   * The augmentation pipeline instance
+   */
+  pipeline: AugmentationPipeline;
+
+  /**
+   * Map of loaded augmentations by type
+   */
+  augmentations: Map<AugmentationType, IAugmentation[]>;
+
+  /**
+   * Any errors that occurred during loading
+   */
+  errors: Error[];
+}
+
+/**
+ * Loads augmentation plugins from external npm packages
+ * 
+ * @param plugins Array of plugin configurations
+ * @param options Options for loading plugins
+ * @returns Promise that resolves to the plugin load result
+ */
+export async function loadPlugins(
+  plugins: PluginConfig[],
+  options: PluginLoaderOptions = {}
+): Promise<PluginLoadResult> {
+  // Merge options with defaults
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  
+  // Use the default pipeline or create a new one
+  const pipeline = opts.useDefaultPipeline ? augmentationPipeline : new AugmentationPipeline();
+  
+  // Track loaded augmentations by type
+  const loadedAugmentations = new Map<AugmentationType, IAugmentation[]>();
+  const errors: Error[] = [];
+
+  // Group plugins by type according to the specified order
+  const pluginsByType = new Map<AugmentationType, PluginConfig[]>();
+  
+  // Initialize the map with empty arrays for each type
+  for (const type of opts.augmentationOrder!) {
+    pluginsByType.set(type, []);
+    loadedAugmentations.set(type, []);
+  }
+  
+  // Group plugins by type
+  for (const pluginConfig of plugins) {
+    const type = pluginConfig.type || AugmentationType.SENSE; // Default to SENSE if not specified
+    const typePlugins = pluginsByType.get(type) || [];
+    typePlugins.push(pluginConfig);
+    pluginsByType.set(type, typePlugins);
+  }
+  
+  // Load plugins in the specified order
+  for (const type of opts.augmentationOrder!) {
+    const typePlugins = pluginsByType.get(type) || [];
+    
+    for (const pluginConfig of typePlugins) {
+      try {
+        // Import the plugin module
+        const pluginModule = await import(pluginConfig.plugin);
+        
+        // Get the default export or the named export that matches the plugin name
+        const PluginClass = pluginModule.default || pluginModule[pluginConfig.plugin.split('/').pop()!];
+        
+        if (!PluginClass) {
+          throw new Error(`Could not find a valid export in plugin ${pluginConfig.plugin}`);
+        }
+        
+        // Instantiate the plugin with the provided configuration
+        const plugin = new PluginClass(pluginConfig.config);
+        
+        // Register the plugin with the pipeline
+        pipeline.register(plugin);
+        
+        // Add to the loaded augmentations map
+        const typeAugmentations = loadedAugmentations.get(type) || [];
+        typeAugmentations.push(plugin);
+        loadedAugmentations.set(type, typeAugmentations);
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        errors.push(err);
+        console.error(`Error loading plugin ${pluginConfig.plugin}:`, err);
+      }
+    }
+  }
+  
+  // Initialize the augmentations if requested
+  if (opts.initializeAfterLoading) {
+    try {
+      await pipeline.initialize();
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      errors.push(err);
+      console.error('Error initializing augmentations:', err);
+    }
+  }
+  
+  return {
+    pipeline,
+    augmentations: loadedAugmentations,
+    errors
+  };
+}
+
+/**
+ * Configures and starts the augmentation pipeline with the specified plugins
+ * 
+ * @param plugins Array of plugin configurations
+ * @param options Options for loading plugins
+ * @returns Promise that resolves to the plugin load result
+ */
+export async function configureAndStartPipeline(
+  plugins: PluginConfig[],
+  options: PluginLoaderOptions = {}
+): Promise<PluginLoadResult> {
+  // Load the plugins
+  const result = await loadPlugins(plugins, {
+    ...options,
+    initializeAfterLoading: true
+  });
+  
+  return result;
+}
+
+/**
+ * Creates a plugin configuration for a sense augmentation
+ * 
+ * @param plugin The plugin module name or path
+ * @param config Optional configuration to pass to the plugin
+ * @returns Plugin configuration
+ */
+export function createSensePluginConfig(
+  plugin: string,
+  config?: Record<string, any>
+): PluginConfig {
+  return {
+    plugin,
+    config,
+    type: AugmentationType.SENSE
+  };
+}
+
+/**
+ * Creates a plugin configuration for a conduit augmentation
+ * 
+ * @param plugin The plugin module name or path
+ * @param config Optional configuration to pass to the plugin
+ * @returns Plugin configuration
+ */
+export function createConduitPluginConfig(
+  plugin: string,
+  config?: Record<string, any>
+): PluginConfig {
+  return {
+    plugin,
+    config,
+    type: AugmentationType.CONDUIT
+  };
+}

src/storage/fileSystemStorage.ts

@@ -451,4 +451,113 @@ export class FileSystemStorage implements StorageAdapter {
     }
     return obj
   }
+
+  /**
+   * Get information about storage usage and capacity
+   */
+  public async getStorageStatus(): Promise<{
+    type: string;
+    used: number;
+    quota: number | null;
+    details?: Record<string, any>;
+  }> {
+    await this.ensureInitialized()
+
+    try {
+      // Calculate the total size of all files in the storage directories
+      let totalSize = 0
+
+      // Helper function to calculate directory size
+      const calculateDirSize = async (dirPath: string): Promise<number> => {
+        let size = 0
+        try {
+          const files = await fs.promises.readdir(dirPath)
+
+          for (const file of files) {
+            const filePath = path.join(dirPath, file)
+            const stats = await fs.promises.stat(filePath)
+
+            if (stats.isDirectory()) {
+              size += await calculateDirSize(filePath)
+            } else {
+              size += stats.size
+            }
+          }
+        } catch (error) {
+          console.warn(`Error calculating size for ${dirPath}:`, error)
+        }
+        return size
+      }
+
+      // Calculate size for each directory
+      const nodesDirSize = await calculateDirSize(this.nodesDir)
+      const edgesDirSize = await calculateDirSize(this.edgesDir)
+      const metadataDirSize = await calculateDirSize(this.metadataDir)
+
+      totalSize = nodesDirSize + edgesDirSize + metadataDirSize
+
+      // Get filesystem information
+      let quota = null
+      let details = {}
+
+      try {
+        // Try to get disk space information
+        const stats = await fs.promises.statfs(this.rootDir)
+        if (stats) {
+          const availableSpace = stats.bavail * stats.bsize
+          const totalSpace = stats.blocks * stats.bsize
+
+          quota = totalSpace
+          details = {
+            availableSpace,
+            totalSpace,
+            freePercentage: (availableSpace / totalSpace) * 100
+          }
+        }
+      } catch (error) {
+        console.warn('Unable to get filesystem stats:', error)
+        // If statfs is not available, try to use df command on Unix-like systems
+        try {
+          const { exec } = await import('child_process')
+          const util = await import('util')
+          const execPromise = util.promisify(exec)
+
+          const { stdout } = await execPromise(`df -k "${this.rootDir}"`)
+          const lines = stdout.trim().split('\n')
+          if (lines.length > 1) {
+            const parts = lines[1].split(/\s+/)
+            if (parts.length >= 4) {
+              const totalKB = parseInt(parts[1], 10)
+              const usedKB = parseInt(parts[2], 10)
+              const availableKB = parseInt(parts[3], 10)
+
+              quota = totalKB * 1024
+              details = {
+                availableSpace: availableKB * 1024,
+                totalSpace: totalKB * 1024,
+                freePercentage: (availableKB / totalKB) * 100
+              }
+            }
+          }
+        } catch (dfError) {
+          console.warn('Unable to get disk space using df command:', dfError)
+        }
+      }
+
+      return {
+        type: 'filesystem',
+        used: totalSize,
+        quota,
+        details
+      }
+    } catch (error) {
+      console.error('Failed to get storage status:', error)
+      return {
+        type: 'filesystem',
+        used: 0,
+        quota: null,
+        details: { error: String(error) }
+      }
+    }
+  }
 }

src/storage/opfsStorage.ts

@@ -498,6 +498,89 @@ export class OPFSStorage implements StorageAdapter {
     }
     return obj
   }
+
+  /**
+   * Get information about storage usage and capacity
+   */
+  public async getStorageStatus(): Promise<{
+    type: string;
+    used: number;
+    quota: number | null;
+    details?: Record<string, any>;
+  }> {
+    await this.ensureInitialized()
+
+    try {
+      // Calculate the total size of all files in the storage directories
+      let totalSize = 0
+
+      // Helper function to calculate directory size
+      const calculateDirSize = async (dirHandle: FileSystemDirectoryHandle): Promise<number> => {
+        let size = 0
+        try {
+          // @ts-ignore - TypeScript doesn't recognize FileSystemDirectoryHandle.entries() properly
+          for await (const [name, handle] of dirHandle.entries()) {
+            if (handle.kind === 'file') {
+              const file = await handle.getFile()
+              size += file.size
+            } else if (handle.kind === 'directory') {
+              size += await calculateDirSize(handle)
+            }
+          }
+        } catch (error) {
+          console.warn(`Error calculating size for directory:`, error)
+        }
+        return size
+      }
+
+      // Calculate size for each directory
+      if (this.nodesDir) {
+        totalSize += await calculateDirSize(this.nodesDir)
+      }
+      if (this.edgesDir) {
+        totalSize += await calculateDirSize(this.edgesDir)
+      }
+      if (this.metadataDir) {
+        totalSize += await calculateDirSize(this.metadataDir)
+      }
+
+      // Get storage quota information using the Storage API
+      let quota = null
+      let details: Record<string, any> = {
+        isPersistent: await this.isPersistent()
+      }
+
+      try {
+        if (navigator.storage && navigator.storage.estimate) {
+          const estimate = await navigator.storage.estimate()
+          quota = estimate.quota || null
+          details = {
+            ...details,
+            usage: estimate.usage,
+            quota: estimate.quota,
+            freePercentage: estimate.quota ? ((estimate.quota - (estimate.usage || 0)) / estimate.quota) * 100 : null
+          }
+        }
+      } catch (error) {
+        console.warn('Unable to get storage estimate:', error)
+      }
+
+      return {
+        type: 'opfs',
+        used: totalSize,
+        quota,
+        details
+      }
+    } catch (error) {
+      console.error('Failed to get storage status:', error)
+      return {
+        type: 'opfs',
+        used: 0,
+        quota: null,
+        details: { error: String(error) }
+      }
+    }
+  }
 }
 
 /**
@@ -693,6 +776,114 @@ export class MemoryStorage implements StorageAdapter {
     this.edges.clear()
     this.metadata.clear()
   }
+
+  /**
+   * Get information about storage usage and capacity
+   */
+  public async getStorageStatus(): Promise<{
+    type: string;
+    used: number;
+    quota: number | null;
+    details?: Record<string, any>;
+  }> {
+    try {
+      // Estimate the size of data in memory
+      let totalSize = 0
+
+      // Helper function to estimate object size in bytes
+      const estimateSize = (obj: any): number => {
+        if (obj === null || obj === undefined) return 0
+
+        const type = typeof obj
+
+        // Handle primitive types
+        if (type === 'number') return 8
+        if (type === 'string') return obj.length * 2
+        if (type === 'boolean') return 4
+
+        // Handle arrays and objects
+        if (Array.isArray(obj)) {
+          return obj.reduce((size, item) => size + estimateSize(item), 0)
+        }
+
+        if (type === 'object') {
+          if (obj instanceof Map) {
+            let mapSize = 0
+            for (const [key, value] of obj.entries()) {
+              mapSize += estimateSize(key) + estimateSize(value)
+            }
+            return mapSize
+          }
+
+          if (obj instanceof Set) {
+            let setSize = 0
+            for (const item of obj) {
+              setSize += estimateSize(item)
+            }
+            return setSize
+          }
+
+          // Regular object
+          return Object.entries(obj).reduce(
+            (size, [key, value]) => size + key.length * 2 + estimateSize(value),
+            0
+          )
+        }
+
+        return 0
+      }
+
+      // Estimate size of nodes
+      for (const node of this.nodes.values()) {
+        totalSize += estimateSize(node)
+      }
+
+      // Estimate size of edges
+      for (const edge of this.edges.values()) {
+        totalSize += estimateSize(edge)
+      }
+
+      // Estimate size of metadata
+      for (const meta of this.metadata.values()) {
+        totalSize += estimateSize(meta)
+      }
+
+      // Get memory information if available
+      let quota = null
+      let details: Record<string, any> = {
+        nodesCount: this.nodes.size,
+        edgesCount: this.edges.size,
+        metadataCount: this.metadata.size
+      }
+
+      // Try to get memory information if in a browser environment
+      if (typeof window !== 'undefined' && (window as any).performance && (window as any).performance.memory) {
+        const memory = (window as any).performance.memory
+        quota = memory.jsHeapSizeLimit
+        details = {
+          ...details,
+          totalJSHeapSize: memory.totalJSHeapSize,
+          usedJSHeapSize: memory.usedJSHeapSize,
+          jsHeapSizeLimit: memory.jsHeapSizeLimit
+        }
+      }
+
+      return {
+        type: 'memory',
+        used: totalSize,
+        quota,
+        details
+      }
+    } catch (error) {
+      console.error('Failed to get memory storage status:', error)
+      return {
+        type: 'memory',
+        used: 0,
+        quota: null,
+        details: { error: String(error) }
+      }
+    }
+  }
 }
 
 /**